From pypy.commits at gmail.com  Mon Aug  1 04:47:01 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 01:47:01 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: merge heads
Message-ID: <579f0c85.56421c0a.23487.5da5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5659:15244739241e
Date: 2016-08-01 10:46 +0200
http://bitbucket.org/pypy/extradoc/changeset/15244739241e/

Log:	merge heads

diff --git a/blog/draft/revdb.rst b/blog/draft/revdb.rst
--- a/blog/draft/revdb.rst
+++ b/blog/draft/revdb.rst
@@ -286,7 +286,8 @@
     (19422)$ bcontinue
     [searching 19325..19422]
 
-    updating watchpoint value: $0.value => RuntimeError: '$0' refers to an object created later in time
+    updating watchpoint value: $0.value => RuntimeError:
+            '$0' refers to an object created later in time
     Reverse-hit watchpoint 1: $0.value
     File "/tmp/x.py", line 6 in <module>:
       import os
diff --git a/talk/ep2016/Makefile b/talk/ep2016/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/ep2016/Makefile
@@ -0,0 +1,6 @@
+slides.pdf: slides.tex author.latex
+	pdflatex $<
+
+slides.tex: slides.rst
+	rst2beamer.py slides.rst > slides.tex
+	sed 's/\\date{}/\\input{author.latex}/' -i slides.tex || exit
diff --git a/talk/ep2016/author.latex b/talk/ep2016/author.latex
new file mode 100644
--- /dev/null
+++ b/talk/ep2016/author.latex
@@ -0,0 +1,7 @@
+\definecolor{rrblitbackground}{rgb}{0.4, 0.0, 0.0}
+
+\title[CFFI and PyPy]{CFFI and PyPy}
+\author[Armin Rigo]{Armin Rigo}
+
+\institute{EuroPython 2016}
+\date{July 2016}
diff --git a/talk/ep2016/slides.rst b/talk/ep2016/slides.rst
new file mode 100644
--- /dev/null
+++ b/talk/ep2016/slides.rst
@@ -0,0 +1,576 @@
+====================================================
+CFFI (and PyPy)
+====================================================
+
+.. raw:: latex
+
+   \catcode`\|=13
+   \def|{\hskip 1cm}
+
+   \let\foobarbaz=>
+   \catcode`\>=13
+   \def>{\foobarbaz\relax}
+
+ 
+
+CFFI
+====
+
+* created in 2012
+
+* successful project according to PyPI
+
+* 3.4 million downloads for January
+
+* total 22.3 millions, 25th place on `pypi-ranking.info`
+
+  - Django is 31st
+
+* some high-visibility projects have switched to it (Cryptography)
+
+
+PyPy
+====
+
+* success: harder to say for sure
+
+* more later
+
+
+CFFI
+====
+
+
+
+CFFI
+====
+
+* call C from Python
+
+* CFFI = C Foreign Function Interface
+
+* shares ideas from Cython, ctypes, LuaJIT's FFI, SWIG...
+
+
+CFFI demo
+=========
+
+::
+
+ |    $ man getpwnam
+
+ | 
+
+ |    SYNOPSIS
+
+ |    |      #include <sys/types.h>
+
+ |    |      #include <pwd.h>
+
+ |    |
+
+ |    |      struct passwd *getpwnam(const char *);
+
+
+CFFI demo
+=========
+
+::
+
+ |       .
+
+ |       .
+
+ |       .
+
+ |       The passwd structure is defined in <pwd.h>
+
+ |       as follows:
+
+ |
+ 
+ |       struct passwd {
+
+ |       |      char   *pw_name;       /* username */
+
+ |       |      char   *pw_passwd;     /* user password */
+ 
+ |       |      uid_t   pw_uid;        /* user ID */
+ 
+ |       .
+
+ |       .
+
+ |       .
+   
+
+CFFI demo
+=========
+
+::
+
+ |  import cffi
+
+ |  ffibuilder = cffi.FFI()
+
+ |
+
+ |  ffibuilder.cdef("""
+
+ |  |     typedef int... uid_t;
+ 
+ |  |     struct passwd {
+ 
+ |  |     |     uid_t pw_uid;
+ 
+ |  |     |     ...;
+ 
+ |  |     };
+ 
+ |  |     struct passwd *getpwnam(const char *);
+ 
+ |  """)
+
+
+CFFI demo
+=========
+
+::
+
+ | ffibuilder.set_source("_pwuid_cffi", """
+ 
+ | |    #include <sys/types.h>
+ 
+ | |    #include <pwd.h>
+ 
+ | """)
+
+ |
+ 
+ | ffibuilder.compile()
+
+ |
+
+... and put that in pwuid_build.py
+
+
+CFFI demo
+=========
+
+::
+
+ | python pwuid_build.py
+
+ |
+
+creates _pwuid_cffi.so
+
+
+CFFI demo
+=========
+
+::
+
+  from _pwuid_cffi import lib
+
+  print lib.getpwnam("username").pw_uid
+
+* That's all folks
+
+
+CFFI demo
+=========
+
+::
+
+  from _pwuid_cffi import ffi, lib
+
+* ``lib`` gives access to all functions from the cdef
+
+  - like ``lib.getpwnam()``
+
+* ``ffi`` gives access to a few general helpers
+
+
+ffibuilder.cdef()
+=====================
+
+::
+
+ |   ffibuilder.cdef("""
+ 
+ |   |   int foo1(int a, int b);
+
+ |   |
+
+ |   |   typedef ... Window;
+ 
+ |   |   Window *make_window(int w, int h);
+ 
+ |   |   void hide_window(Window *);
+ 
+ |   """)
+
+
+ffi.new()
+=========
+
+::
+    
+ |   >>> p = ffi.new("char[]", "Some string")
+ 
+ |   >>> p
+ 
+ |   <cdata 'char[]' owning 12 bytes>
+
+ |
+ 
+ |   >>> p[1]
+ 
+ |   'o'
+
+ |
+ 
+ |   >>> q = lib.getpwnam(p)
+ 
+ |   >>> q
+ 
+ |   <cdata 'struct passwd *' 0x12345678>
+
+ |
+  
+ |   >>> q.pw_uid
+ 
+ |   500
+
+ffi.cast()
+==========
+
+::
+
+ |   >>> q = lib.getpwnam("root")
+
+ |   >>> q
+
+ |   <cdata 'struct passwd *' 0x12345678>
+
+ |
+
+ |   >>> ffi.cast("void *", q)
+
+ |   <cdata 'void *' 0x12345678>
+
+ |
+
+ |   >>> int(ffi.cast("intptr_t", q))
+
+ |   305419896
+
+ |   >>> hex(_)
+
+ |   0x12345678
+
+
+ffi.string()
+============
+
+::
+
+ |   >>> p
+
+ |   <cdata 'struct passwd *' 0x12345678>
+
+ |
+
+ |   >>> p.pw_uid
+
+ |   500
+
+ |
+
+ |   >>> p.pw_name
+
+ |   <cdata 'char *' 0x5234abcd>
+
+ |
+
+ |   >>> ffi.string(p.pw_name)
+
+ |   "username"
+
+
+ffi.new_handle()
+================
+
+::
+
+ |   >>> x = X()
+
+ |   >>> h1 = ffi.new_handle(x)
+ 
+ |   >>> h1
+ 
+ |   <cdata 'void *' handle to
+
+ |   | | | | <X object at 0x123456>>
+ 
+ |   >>> lib.save_away(h1)
+ 
+ |
+ 
+ |   >>> h2 = lib.fish_again()
+ 
+ |   >>> h2
+ 
+ |   <cdata 'void *' 0x87654321>
+
+ |
+
+ |   >>> ffi.from_handle(h2)
+ 
+ |   <X object at 0x123456>
+
+
+CFFI
+====
+
+* supports more or less the whole C
+
+* there is more than this short introduction suggests
+
+
+CFFI
+====
+
+* in real life, you want to provide a Pythonic API to a C library
+
+* you write Python functions and classes implementing it
+
+* all CFFI objects like ``<cdata 'foo *'>`` are hidden inside
+
+
+CFFI
+====
+
+* other use cases:
+
+  - call C code that you write yourself, not a separate C library
+
+  - API versus ABI mode: can also run in a ctypes-like way if
+    you don't want to depend on any C compiler at all
+
+* support for "embedding" Python inside some other non-Python program
+
+  - now you really never need the CPython C API any more
+  
+
+CFFI
+====
+
+* see the docs: http://cffi.readthedocs.org/
+
+
+
+PyPy
+====
+
+
+PyPy
+====
+
+* a Python interpreter
+
+* different from the standard, which is CPython
+
+* main goal of PyPy: speed
+
+
+PyPy
+====
+
+::
+
+ |   $ pypy
+
+ |  Python 2.7.10 (7e8df3df9641, Jun 28 2016)
+
+ |  [PyPy 5.3.1 with GCC 6.1.1] on linux2
+ 
+ |  Type "help", "copyright", "credits" or
+
+ |  >>>> 2+3
+
+ |  5
+
+ |  >>>>
+
+
+PyPy
+====
+
+* run ``pypy my_program.py`` instead of ``python my_program.py``
+
+* contains a JIT compiler
+
+
+PyPy: Garbage Collection
+========================
+
+* "**moving,** generational, incremental GC"
+
+* objects don't have reference counters
+
+* allocated in a "nursery"
+
+* when nursery full, surviving objects are moved out
+
+* usually works on nursery objects only (fast), but rarely also perform
+  a full GC
+
+
+PyPy: C extensions
+==================
+
+* PyPy works great for running Python
+
+* less great when there are CPython C extension modules involved
+
+* not directly possible: we have moving, non-reference-counted objects,
+  and the C code expects non-moving, reference-counted objects
+
+
+PyPy: C extensions
+==================
+
+* PyPy has still some support for them, called its ``cpyext`` module
+
+* emulate all objects for C extensions with a shadow, non-movable,
+  reference-counted object
+
+* ``cpyext`` is slow
+
+* it should "often" work even with large libraries
+  (e.g. ``numpy`` support is mostly there)
+
+
+PyPy: ad
+========
+
+* but, hey, if you need performance out of Python and don't rely
+  critically on C extension modules, then give PyPy a try
+
+  - typical area where it works well: web services
+
+
+CPython C API: the problem
+==========================
+
+* CPython comes with a C API
+
+* very large number of functions
+
+* assumes objects don't move
+
+* assumes a "reference counting" model
+
+
+CPython C API
+=============
+
+* actually, the API is some large subset of the functions inside
+  CPython itself
+
+
+CPython C API
+=============
+
+* easy to use from C
+
+* historically, part of the success of Python
+
+
+CPython C API
+=============
+
+* further successful tools build on top of that API:
+
+  - SWIG
+  - Cython
+  - and other binding generators
+  - now CFFI
+
+
+CFFI
+====
+
+* but CFFI is a bit different
+
+  - it does not expose any part of the CPython C API
+
+  - everything is done with a minimal API on the ``ffi`` object
+    which is closer to C
+
+    - ``ffi.cast()``, ``ffi.new()``, etc.
+
+  - that means it can be directly ported
+
+
+CFFI and PyPy
+=============
+
+* we have a PyPy version of CFFI
+
+* the demos I have given above work equally well on CPython or on PyPy
+
+* (supporting PyPy was part of the core motivation behind CFFI)
+
+
+CFFI: performance
+=================
+
+* in PyPy, JIT compiler speeds up calls, so it's very fast
+
+* in CPython, it doesn't occur, but it is still reasonable when
+  compared with alternatives
+
+* main issue is that we write more code in Python with CFFI,
+  which makes it slower on CPython---but not really on PyPy
+
+
+CFFI: summary
+=============
+
+* call C from Python
+
+* works natively on CPython and on PyPy
+
+  - and easy to port to other Python implementations
+
+* supports CPython 2.6, 2.7, 3.2 to 3.5, and
+  is integrated with PyPy
+
+
+CFFI
+====
+
+* independent on the particular details of the Python implementation
+
+  - using CFFI, you call C functions and manipulate C-pointer-like
+    objects directly from Python
+
+  - you do in Python all logic involving Python objects
+
+  - there are no (official) ways around this API to call the CPython C
+    API, and none are needed
+
+
+CFFI
+====
+
+* two reasons to switch to it ``:-)``
+
+  - easy and cool
+
+  - better supported on non-CPython implementations
+
+* http://cffi.readthedocs.org/

From pypy.commits at gmail.com  Mon Aug  1 04:46:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 01:46:59 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: draft for jitlog blog post
Message-ID: <579f0c83.8411c20a.bcd4e.fdc1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5658:8bb00903feae
Date: 2016-08-01 10:19 +0200
http://bitbucket.org/pypy/extradoc/changeset/8bb00903feae/

Log:	draft for jitlog blog post

diff --git a/blog/draft/new-jit-log.rst b/blog/draft/new-jit-log.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/new-jit-log.rst
@@ -0,0 +1,54 @@
+PyPy's Toolbox got a new Hammer &#x1f528;
+=======
+
+Tools, tools, tools! It seems that PyPy cannot get enough of them!
+In the last winter sprint (Leysin) covered the current tool for observing interals of the JIT compiler (JitViewer). VMProf at that time already proved that it is a good tool for CPU profiles. We are happy to release a new version of VMProf incooperating a rewritten version of JitViewer.
+
+The old logging format, is a hard to maintain plain text logging facility. Frequent changes often broke internal tools most notably the JITViewer. A second bad taste is that the logging output of a long running program takes a lot of space.
+
+Our new binary format encodes data densly, makes use of some compression (gzip) and tries to remove repetition where possible. On top of that protocol supports versioning and can be extended easily. And! *durms* you do not need to install JitViewer yourself anymore. The whole system moved to vmprof.com and you can use it any time free of charge.
+
+Sounds great. But what can you do with it? Here are two examples useful for a PyPy user:
+
+PyPy crashed? Did you discover a bug?
+-------------------
+
+For some hard to find bugs it is often necessary to look at the compiled code. The old procedure often required to upload a plain text file which was hard to parse and to look through. 
+
+The new way to share a crash report is to install vmprof and execute either of the two commands:
+
+```
+# this program does not crash, but has some weird behaviour
+$ pypy -m jitlog --web <your program args>
+...
+PyPy Jitlog: http://vmprof.com/#/<hash>
+# this program segfaults
+$ pypy -m jitlog -o /tmp/log <your program args>
+...
+<Segfault>
+$ pypy -m jitlog --upload /tmp/log
+PyPy Jitlog: http://vmprof.com/#/<hash>
+```
+
+Providing the link in the bug report enables PyPy developers browse and identify potential issues.
+
+Speed issues
+------------
+
+VMProf is a great tool to find out hot spots that consume a lot of time in your program. As soon as you have idenified code that runs slow, you can switch to jitlog and maybe pin point certain aspects that do not behave as expected. You will find not only the overview, but are also able to browse the generated code. If you cannot make sense of that all you can just share the link with us and we can have a look at the compiled code.
+
+Future direction
+----------------
+
+We hope that the new release will help both PyPy developers and PyPy users resolve potential issues and easily point them out.
+
+Here are a few ideas what might come in the next few releases ().
+
+* Extend vmprof.com to be able to query vmprof/jitlog. Some times it is interesting to search for specific patterns the compiler produced. An example for vmprof: 'methods.callsites() > 5' and for the jitlog would be 'traces.contains('call_assembler').hasbridge('*my_func_name*')'
+
+* Combination of CPU profiles and the JITLOG (Sadly did not make it into the current release)
+
+* Extend the jitlog to capture the information of the optimization stage
+
+plan_rich and the PyPy team
+

From pypy.commits at gmail.com  Mon Aug  1 08:46:21 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 05:46:21 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: impl. vec_float_xor (missing
 change)
Message-ID: <579f449d.d4e01c0a.f1ec7.c83f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r85954:7510e65f6e05
Date: 2016-08-01 14:45 +0200
http://bitbucket.org/pypy/pypy/changeset/7510e65f6e05/

Log:	impl. vec_float_xor (missing change)

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -274,6 +274,8 @@
     def genop_vec_int_xor(self, op, arglocs, resloc):
         self.mc.PXOR(resloc, arglocs[0])
 
+    genop_vec_float_xor = genop_vec_int_xor
+
     genop_vec_float_arith = """
     def genop_vec_float_{type}(self, op, arglocs, resloc):
         loc0, loc1, itemsize_loc = arglocs
@@ -639,6 +641,7 @@
     consider_vec_int_and = consider_vec_logic
     consider_vec_int_or = consider_vec_logic
     consider_vec_int_xor = consider_vec_logic
+    consider_vec_float_xor = consider_vec_logic
     del consider_vec_logic
 
     def consider_vec_pack_i(self, op):

From pypy.commits at gmail.com  Mon Aug  1 10:23:28 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 07:23:28 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: correct the scaling for
 vec_load/store
Message-ID: <579f5b60.56421c0a.23487.f41c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r85955:84c8a9345e22
Date: 2016-08-01 16:22 +0200
http://bitbucket.org/pypy/pypy/changeset/84c8a9345e22/

Log:	correct the scaling for vec_load/store

diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py
--- a/rpython/jit/backend/llsupport/vector_ext.py
+++ b/rpython/jit/backend/llsupport/vector_ext.py
@@ -2,7 +2,7 @@
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr, ArrayDescr)
 from rpython.rlib.objectmodel import specialize, always_inline
-from rpython.jit.metainterp.history import (VECTOR, FLOAT, INT)
+from rpython.jit.metainterp.history import (VECTOR, FLOAT, INT, ConstInt)
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.optimizeopt.schedule import (forwarded_vecinfo,
         failnbail_transformation)
@@ -72,7 +72,7 @@
         self.argument_restrictions = argument_restris
 
     def check_operation(self, state, pack, op):
-        pass
+        return None
 
     def crop_vector(self, op, newsize, size):
         return newsize, size
@@ -111,19 +111,25 @@
 class LoadRestrict(OpRestrict):
     def check_operation(self, state, pack, op):
         opnum = op.getopnum()
+        descr = op.getdescr()
+        if not we_are_translated() and not isinstance(descr, ArrayDescr):
+            itemsize = descr.get_item_size_in_bytes()
+            ofs = 0
+        else:
+            itemsize, ofs, _ = unpack_arraydescr(op.getdescr())
+        args = [op.getarg(0), op.getarg(1), ConstInt(1), ConstInt(ofs)]
         if rop.is_getarrayitem(opnum) or \
              opnum in (rop.GETARRAYITEM_RAW_I, rop.GETARRAYITEM_RAW_F):
-            descr = op.getdescr()
-            if not we_are_translated() and not isinstance(descr, ArrayDescr):
-                itemsize = descr.get_item_size_in_bytes()
-                ofs = 0
-            else:
-                itemsize, ofs, _ = unpack_arraydescr(op.getdescr())
             index_box = op.getarg(1)
-            _, _, changed, emit = cpu_simplify_scale(state.cpu, index_box, itemsize, ofs)
+            scale, offset, changed, emit = cpu_simplify_scale(state.cpu, index_box, itemsize, ofs)
+            args[2] = ConstInt(scale)
+            args[3] = ConstInt(offset)
             if emit:
                 state.oplist.append(changed)
-                op.setarg(1, changed)
+                args[1] = changed
+
+        return args
+
 
     def opcount_filling_vector_register(self, op, vec_reg_size):
         assert rop.is_primitive_load(op.opnum)
@@ -136,18 +142,22 @@
 
     def check_operation(self, state, pack, op):
         opnum = op.getopnum()
+        descr = op.getdescr()
+        if not we_are_translated() and not isinstance(descr, ArrayDescr):
+            itemsize = descr.get_item_size_in_bytes()
+            ofs = 0
+        else:
+            itemsize, ofs, _ = unpack_arraydescr(op.getdescr())
+        args = [op.getarg(0), op.getarg(1), op.getarg(2), ConstInt(1), ConstInt(ofs)]
         if opnum in (rop.SETARRAYITEM_GC, rop.SETARRAYITEM_RAW):
-            descr = op.getdescr()
-            if not we_are_translated() and not isinstance(descr, ArrayDescr):
-                itemsize = descr.get_item_size_in_bytes()
-                basesize= 0
-            else:
-                itemsize, basesize, _ = unpack_arraydescr(op.getdescr())
             index_box = op.getarg(1)
-            _, _, changed, emit = cpu_simplify_scale(state.cpu, index_box, itemsize, basesize)
+            scale, offset, changed, emit = cpu_simplify_scale(state.cpu, index_box, itemsize, ofs)
+            args[3] = ConstInt(scale)
+            args[4] = ConstInt(offset)
             if emit:
                 state.oplist.append(changed)
-                op.setarg(1, changed)
+                args[1] = changed
+        return args
 
     def must_crop_vector(self, op, index):
         vecinfo = forwarded_vecinfo(op.getarg(index))
@@ -185,6 +195,7 @@
                 raise NotAVectorizeableLoop()
             if curvecinfo.datatype != datatype:
                 raise NotAVectorizeableLoop()
+        return None
 
 TR_ANY = TypeRestrict()
 TR_ANY_FLOAT = TypeRestrict(FLOAT)
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -150,26 +150,11 @@
 
         not_implemented("reduce sum for %s not impl." % arg)
 
-    # TODO remove
-    #def _genop_vec_getarrayitem(self, op, arglocs, resloc):
-    #    # considers item scale (raw_load does not)
-    #    base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
-    #    scale = get_scale(size_loc.value)
-    #    src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale)
-    #    self._vec_load(resloc, src_addr, integer_loc.value,
-    #                   size_loc.value, aligned_loc.value)
-    #
-    #genop_vec_getarrayitem_raw_i = _genop_vec_getarrayitem
-    #genop_vec_getarrayitem_raw_f = _genop_vec_getarrayitem
-    #
-    #genop_vec_getarrayitem_gc_i = _genop_vec_getarrayitem
-    #genop_vec_getarrayitem_gc_f = _genop_vec_getarrayitem
-
     def _genop_vec_load(self, op, arglocs, resloc):
-        base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
-        src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0)
+        base_loc, ofs_loc, size_loc, scale, ofs, integer_loc = arglocs
+        src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale.value)
         self._vec_load(resloc, src_addr, integer_loc.value,
-                       size_loc.value, aligned_loc.value)
+                       size_loc.value, False)
 
     genop_vec_load_i = _genop_vec_load
     genop_vec_load_f = _genop_vec_load
@@ -187,23 +172,12 @@
             elif itemsize == 8:
                 self.mc.MOVUPD(resloc, src_addr)
 
-    # TODO remove
-    #def _genop_discard_vec_setarrayitem(self, op, arglocs):
-    #    # considers item scale (raw_store does not)
-    #    base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
-    #    scale = get_scale(size_loc.value)
-    #    dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, scale)
-    #    self._vec_store(dest_loc, value_loc, integer_loc.value,
-    #                    size_loc.value, aligned_loc.value)
-
-    #genop_discard_vec_setarrayitem_raw = _genop_discard_vec_setarrayitem
-    #genop_discard_vec_setarrayitem_gc = _genop_discard_vec_setarrayitem
-
     def genop_discard_vec_store(self, op, arglocs):
-        base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
-        dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0)
+        base_loc, ofs_loc, value_loc, size_loc, scale,\
+                baseofs, integer_loc = arglocs
+        dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, scale.value)
         self._vec_store(dest_loc, value_loc, integer_loc.value,
-                        size_loc.value, aligned_loc.value)
+                        size_loc.value, False)
 
     @always_inline
     def _vec_store(self, dest_loc, value_loc, integer, itemsize, aligned):
@@ -550,20 +524,17 @@
         assert isinstance(descr, ArrayDescr)
         assert not descr.is_array_of_pointers() and \
                not descr.is_array_of_structs()
-        itemsize, ofs, _ = unpack_arraydescr(descr)
+        itemsize, _, _ = unpack_arraydescr(descr)
         integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
-        aligned = False
         args = op.getarglist()
+        scale = get_scale(op.getarg(2).getint())
+        ofs = op.getarg(3).getint()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
         ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
         result_loc = self.force_allocate_reg(op)
-        self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs),
-                          imm(integer), imm(aligned)], result_loc)
+        self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(scale),
+                          imm(ofs), imm(integer)], result_loc)
 
-    #consider_vec_getarrayitem_raw_i = _consider_vec_getarrayitem
-    #consider_vec_getarrayitem_raw_f = _consider_vec_getarrayitem
-    #consider_vec_getarrayitem_gc_i = _consider_vec_getarrayitem
-    #consider_vec_getarrayitem_gc_f = _consider_vec_getarrayitem
     consider_vec_load_i = _consider_vec_load
     consider_vec_load_f = _consider_vec_load
 
@@ -573,20 +544,18 @@
         assert isinstance(descr, ArrayDescr)
         assert not descr.is_array_of_pointers() and \
                not descr.is_array_of_structs()
-        itemsize, ofs, _ = unpack_arraydescr(descr)
+        itemsize, _, _ = unpack_arraydescr(descr)
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
         value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
         ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+        scale = get_scale(op.getarg(3).getint())
+        ofs = op.getarg(4).getint()
 
-        integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
-        aligned = False
-        self.perform_discard(op, [base_loc, ofs_loc, value_loc,
-                                 imm(itemsize), imm(ofs), imm(integer), imm(aligned)])
-
-    #consider_vec_setarrayitem_raw = _consider_vec_setarrayitem
-    #consider_vec_setarrayitem_gc = _consider_vec_setarrayitem
-    #consider_vec_store = _consider_vec_setarrayitem
+        integer = not (descr.is_array_of_floats() or \
+                       descr.getconcrete_type() == FLOAT)
+        self.perform_discard(op, [base_loc, ofs_loc, value_loc, imm(itemsize),
+                                  imm(scale), imm(ofs), imm(integer)])
 
     def consider_vec_arith(self, op):
         lhs = op.getarg(0)
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -220,11 +220,17 @@
     left = pack.leftmost()
     oprestrict = state.cpu.vector_ext.get_operation_restriction(left)
     if oprestrict is not None:
-        oprestrict.check_operation(state, pack, left)
-    args = left.getarglist_copy()
+        newargs = oprestrict.check_operation(state, pack, left)
+        if newargs:
+            args = newargs
+        else:
+            args = left.getarglist_copy()
+    else:
+        args = left.getarglist_copy()
     prepare_arguments(state, oprestrict, pack, args)
     vecop = VecOperation(left.vector, args, left,
                          pack.numops(), left.getdescr())
+
     for i,node in enumerate(pack.operations):
         op = node.getoperation()
         if op.returns_void():
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1077,7 +1077,7 @@
     'GETARRAYITEM_GC/2d/rfi',
     'GETARRAYITEM_RAW/2d/fi',
     'RAW_LOAD/2d/fi',
-    'VEC_LOAD/2d/fi',
+    'VEC_LOAD/4d/fi',
     '_RAW_LOAD_LAST',
 
     'GETINTERIORFIELD_GC/2d/rfi',
@@ -1112,7 +1112,7 @@
     'SETARRAYITEM_GC/3d/n',
     'SETARRAYITEM_RAW/3d/n',
     'RAW_STORE/3d/n',
-    'VEC_STORE/3d/n',
+    'VEC_STORE/5d/n',
     '_RAW_STORE_LAST',
     'SETINTERIORFIELD_GC/3d/n',
     'SETINTERIORFIELD_RAW/3d/n',    # right now, only used by tests

From pypy.commits at gmail.com  Mon Aug  1 10:44:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 07:44:02 -0700 (PDT)
Subject: [pypy-commit] pypy jitlog-exact-source-lines: use offset2lineno as
 proposed by cfbolz
Message-ID: <579f6032.09afc20a.a7367.9390@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: jitlog-exact-source-lines
Changeset: r85956:7467ad8d230f
Date: 2016-08-01 16:41 +0200
http://bitbucket.org/pypy/pypy/changeset/7467ad8d230f/

Log:	use offset2lineno as proposed by cfbolz

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -42,6 +42,22 @@
     from rpython.rlib import rvmprof
     return rvmprof.get_unique_id(bytecode)
 
+ at jit.elidable
+def offset2lineno(bytecode, stopat):
+    # see dis.findlinestarts for an explanation. This function is copied from
+    # rpython/tool/error.py
+    # lnotab is a list of [byte inc, line inc, ...]
+    # even position denote byte increments, odd line increments...
+    tab = bytecode.co_lnotab
+    line = bytecode.co_firstlineno
+    addr = 0
+    for i in range(0, len(tab), 2):
+        addr = addr + ord(tab[i])
+        if addr > stopat:
+            break
+        line = line + ord(tab[i+1])
+    return line
+
 @jl.returns(jl.MP_FILENAME, jl.MP_LINENO,
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
@@ -53,7 +69,8 @@
     name = bytecode.co_name
     if not name:
         name = ""
-    return (bytecode.co_filename, bytecode.co_firstlineno,
+    line = offset2lineno(bytecode, next_instr)
+    return (bytecode.co_filename, line,
             name, intmask(next_instr), opname)
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):

From pypy.commits at gmail.com  Mon Aug  1 11:44:49 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 01 Aug 2016 08:44:49 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: some tweaks and comments
Message-ID: <579f6e71.17a61c0a.70991.1730@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: extradoc
Changeset: r5660:7c07538c059b
Date: 2016-08-01 17:44 +0200
http://bitbucket.org/pypy/extradoc/changeset/7c07538c059b/

Log:	some tweaks and comments

diff --git a/blog/draft/new-jit-log.rst b/blog/draft/new-jit-log.rst
--- a/blog/draft/new-jit-log.rst
+++ b/blog/draft/new-jit-log.rst
@@ -1,12 +1,18 @@
 PyPy's Toolbox got a new Hammer &#x1f528;
 =======
 
+.. : XXX the title is very generic
+
+.. : XXX I don't actually like the first paragraph, I think it should be more
+    to the point. eg that things happened at the Leysin sprint doesn't matter much.
+    I would also add links to all the existing tools
+
 Tools, tools, tools! It seems that PyPy cannot get enough of them!
-In the last winter sprint (Leysin) covered the current tool for observing interals of the JIT compiler (JitViewer). VMProf at that time already proved that it is a good tool for CPU profiles. We are happy to release a new version of VMProf incooperating a rewritten version of JitViewer.
+In the last winter sprint (Leysin) covered the current tool for observing internals of the JIT compiler (JitViewer). VMProf at that time already proved that it is a good tool for CPU profiles. We are happy to release a new version of VMProf incorporating a rewritten version of JitViewer.
 
-The old logging format, is a hard to maintain plain text logging facility. Frequent changes often broke internal tools most notably the JITViewer. A second bad taste is that the logging output of a long running program takes a lot of space.
+The old logging format was a hard to maintain plain text logging facility. Frequent changes often broke internal tools, most notably the JITViewer. Another problem was that the logging output of a long running program took a lot of disk space.
 
-Our new binary format encodes data densly, makes use of some compression (gzip) and tries to remove repetition where possible. On top of that protocol supports versioning and can be extended easily. And! *durms* you do not need to install JitViewer yourself anymore. The whole system moved to vmprof.com and you can use it any time free of charge.
+Our new binary format encodes data densly, makes use of some compression (gzip) and tries to remove repetition where possible. On top of that protocol supports versioning and can be extended easily. And *drumroll* you do not need to install JitViewer yourself anymore! The whole system moved to vmprof.com and you can use it any time.
 
 Sounds great. But what can you do with it? Here are two examples useful for a PyPy user:
 
@@ -15,7 +21,7 @@
 
 For some hard to find bugs it is often necessary to look at the compiled code. The old procedure often required to upload a plain text file which was hard to parse and to look through. 
 
-The new way to share a crash report is to install vmprof and execute either of the two commands:
+The new way to share a crash report is to install the ``vmprof`` module from PyPi and execute either of the two commands:
 
 ```
 # this program does not crash, but has some weird behaviour
@@ -42,13 +48,13 @@
 
 We hope that the new release will help both PyPy developers and PyPy users resolve potential issues and easily point them out.
 
-Here are a few ideas what might come in the next few releases ().
+Here are a few ideas what might come in the next few releases:
+
+* Combination of CPU profiles and the JITLOG (Sadly did not make it into the current release)
 
 * Extend vmprof.com to be able to query vmprof/jitlog. Some times it is interesting to search for specific patterns the compiler produced. An example for vmprof: 'methods.callsites() > 5' and for the jitlog would be 'traces.contains('call_assembler').hasbridge('*my_func_name*')'
 
-* Combination of CPU profiles and the JITLOG (Sadly did not make it into the current release)
-
 * Extend the jitlog to capture the information of the optimization stage
 
-plan_rich and the PyPy team
+Richard Plangger (plan_rich) and the PyPy team
 

From pypy.commits at gmail.com  Mon Aug  1 12:08:11 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 01 Aug 2016 09:08:11 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: a nicer get_printable_location
Message-ID: <579f73eb.45d11c0a.ea683.20ec@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: reverse-debugger
Changeset: r85958:fa384c882956
Date: 2016-07-14 10:44 +0200
http://bitbucket.org/pypy/pypy/changeset/fa384c882956/

Log:	a nicer get_printable_location

diff --git a/rpython/jit/tl/tla/tla.py b/rpython/jit/tl/tla/tla.py
--- a/rpython/jit/tl/tla/tla.py
+++ b/rpython/jit/tl/tla/tla.py
@@ -60,19 +60,34 @@
 
 # ____________________________________________________________
 
-CONST_INT = 1
-POP       = 2
-ADD       = 3
-RETURN    = 4
-JUMP_IF   = 5
-DUP       = 6
-SUB       = 7
-NEWSTR    = 8
+OPNAMES = []
+HASARG = []
+
+def define_op(name, has_arg=False):
+    globals()[name] = len(OPNAMES)
+    OPNAMES.append(name)
+    HASARG.append(has_arg)
+
+define_op("CONST_INT", True)
+define_op("POP")
+define_op("ADD")
+define_op("RETURN")
+define_op("JUMP_IF", True)
+define_op("DUP")
+define_op("SUB")
+define_op("NEWSTR", True)
+
 
 # ____________________________________________________________
 
 def get_printable_location(pc, bytecode):
-    return str(pc)
+    op = ord(bytecode[pc])
+    name = OPNAMES[op]
+    if HASARG[op]:
+        arg = str(ord(bytecode[pc + 1]))
+    else:
+        arg = ''
+    return "%s: %s %s" % (pc, name, arg)
 
 jitdriver = JitDriver(greens=['pc', 'bytecode'],
                       reds=['self'],

From pypy.commits at gmail.com  Mon Aug  1 12:08:09 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 01 Aug 2016 09:08:09 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: add --jit option to the tla
 target
Message-ID: <579f73e9.85261c0a.3c27.1cac@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: reverse-debugger
Changeset: r85957:49e0ce99e325
Date: 2016-07-14 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/49e0ce99e325/

Log:	add --jit option to the tla target

diff --git a/rpython/jit/tl/tla/targettla.py b/rpython/jit/tl/tla/targettla.py
--- a/rpython/jit/tl/tla/targettla.py
+++ b/rpython/jit/tl/tla/targettla.py
@@ -4,9 +4,16 @@
 
 
 def entry_point(args):
-    """Main entry point of the stand-alone executable:
-    takes a list of strings and returns the exit code.
-    """
+    for i in range(len(argv)):
+        if argv[i] == "--jit":
+            if len(argv) == i + 1:
+                print "missing argument after --jit"
+                return 2
+            jitarg = argv[i + 1]
+            del argv[i:i+2]
+            jit.set_user_param(jitdriver, jitarg)
+            break
+
     if len(args) < 3:
         print "Usage: %s filename x" % (args[0],)
         return 2
@@ -26,7 +33,7 @@
     return bytecode
 
 def target(driver, args):
-    return entry_point, None
+    return entry_point
 
 # ____________________________________________________________
 

From pypy.commits at gmail.com  Mon Aug  1 12:08:13 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 01 Aug 2016 09:08:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: make int * string work too in RPython
Message-ID: <579f73ed.eeb8c20a.79cf4.1d3f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r85959:2246f93d2550
Date: 2016-08-01 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/2246f93d2550/

Log:	make int * string work too in RPython

diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -591,7 +591,9 @@
 
 class __extend__(pairtype(IntegerRepr, AbstractStringRepr)):
     def rtype_mul((r_int, r_str), hop):
-        return pair(r_str, r_int).rtype_mul(hop)
+        str_repr = r_str.repr
+        v_int, v_str = hop.inputargs(Signed, str_repr)
+        return hop.gendirectcall(r_str.ll.ll_str_mul, v_str, v_int)
     rtype_inplace_mul = rtype_mul
 
 
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -220,11 +220,12 @@
         const = self.const
         def fn(i, mul):
             s = ["", "a", "aba"][i]
-            return s * mul
+            return s * mul + mul * s
         for i in xrange(3):
             for m in [0, 1, 4]:
+                res1 = fn(i, m)
                 res = self.interpret(fn, [i, m])
-                assert self.ll_to_string(res) == fn(i, m)
+                assert self.ll_to_string(res) == res1
 
     def test_is_none(self):
         const = self.const

From pypy.commits at gmail.com  Mon Aug  1 12:11:10 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 01 Aug 2016 09:11:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: add --jit option to the tla target
Message-ID: <579f749e.17a61c0a.70991.21b8@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r85960:dc10853dfd3b
Date: 2016-07-14 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/dc10853dfd3b/

Log:	add --jit option to the tla target

diff --git a/rpython/jit/tl/tla/targettla.py b/rpython/jit/tl/tla/targettla.py
--- a/rpython/jit/tl/tla/targettla.py
+++ b/rpython/jit/tl/tla/targettla.py
@@ -4,9 +4,16 @@
 
 
 def entry_point(args):
-    """Main entry point of the stand-alone executable:
-    takes a list of strings and returns the exit code.
-    """
+    for i in range(len(argv)):
+        if argv[i] == "--jit":
+            if len(argv) == i + 1:
+                print "missing argument after --jit"
+                return 2
+            jitarg = argv[i + 1]
+            del argv[i:i+2]
+            jit.set_user_param(jitdriver, jitarg)
+            break
+
     if len(args) < 3:
         print "Usage: %s filename x" % (args[0],)
         return 2
@@ -26,7 +33,7 @@
     return bytecode
 
 def target(driver, args):
-    return entry_point, None
+    return entry_point
 
 # ____________________________________________________________
 

From pypy.commits at gmail.com  Mon Aug  1 12:11:12 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 01 Aug 2016 09:11:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: a nicer get_printable_location
Message-ID: <579f74a0.a717c20a.804d7.c497@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r85961:e3e2a10c5153
Date: 2016-07-14 10:44 +0200
http://bitbucket.org/pypy/pypy/changeset/e3e2a10c5153/

Log:	a nicer get_printable_location

diff --git a/rpython/jit/tl/tla/tla.py b/rpython/jit/tl/tla/tla.py
--- a/rpython/jit/tl/tla/tla.py
+++ b/rpython/jit/tl/tla/tla.py
@@ -60,19 +60,34 @@
 
 # ____________________________________________________________
 
-CONST_INT = 1
-POP       = 2
-ADD       = 3
-RETURN    = 4
-JUMP_IF   = 5
-DUP       = 6
-SUB       = 7
-NEWSTR    = 8
+OPNAMES = []
+HASARG = []
+
+def define_op(name, has_arg=False):
+    globals()[name] = len(OPNAMES)
+    OPNAMES.append(name)
+    HASARG.append(has_arg)
+
+define_op("CONST_INT", True)
+define_op("POP")
+define_op("ADD")
+define_op("RETURN")
+define_op("JUMP_IF", True)
+define_op("DUP")
+define_op("SUB")
+define_op("NEWSTR", True)
+
 
 # ____________________________________________________________
 
 def get_printable_location(pc, bytecode):
-    return str(pc)
+    op = ord(bytecode[pc])
+    name = OPNAMES[op]
+    if HASARG[op]:
+        arg = str(ord(bytecode[pc + 1]))
+    else:
+        arg = ''
+    return "%s: %s %s" % (pc, name, arg)
 
 jitdriver = JitDriver(greens=['pc', 'bytecode'],
                       reds=['self'],

From pypy.commits at gmail.com  Mon Aug  1 13:05:03 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 10:05:03 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: impl. flush_vector_cc for x86
 using PBLENDVB
Message-ID: <579f813f.68adc20a.8208b.c48b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r85962:7673f44c3693
Date: 2016-08-01 19:04 +0200
http://bitbucket.org/pypy/pypy/changeset/7673f44c3693/

Log:	impl. flush_vector_cc for x86 using PBLENDVB

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -108,9 +108,20 @@
         single_neg_const = '\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80'
         zero_const = '\x00' * 16
         #
+        two_64bit_ones = '\x01\x00\x00\x00\x00\x00\x00\x00' * 2
+        four_32bit_ones = '\x01\x00\x00\x00' * 4
+        eight_16bit_ones = '\x01\x00' * 8
+        sixteen_8bit_ones = '\x01' * 16
+
+
+
+
+
+        #
         data = neg_const + abs_const + \
                single_neg_const + single_abs_const + \
-               zero_const
+               zero_const + sixteen_8bit_ones + eight_16bit_ones + \
+               four_32bit_ones + two_64bit_ones
         datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, [])
         float_constants = datablockwrapper.malloc_aligned(len(data), alignment=16)
         datablockwrapper.done()
@@ -122,6 +133,7 @@
         self.single_float_const_neg_addr = float_constants + 32
         self.single_float_const_abs_addr = float_constants + 48
         self.expand_byte_mask_addr = float_constants + 64
+        self.element_ones = [float_constants + 80 + 16*i for i in range(4)]
 
     def set_extra_stack_depth(self, mc, value):
         if self._is_asmgcc():
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -793,6 +793,7 @@
 
     PTEST_xx = xmminsn('\x66', rex_nw, '\x0F\x38\x17', register(1,8), register(2), '\xC0')
     PBLENDW_xxi = xmminsn('\x66', rex_nw, '\x0F\x3A\x0E', register(1,8), register(2), '\xC0', immediate(3, 'b'))
+    PBLENDVB_xx = xmminsn('\x66', rex_nw, '\x0F\x38\x10', register(1,8), register(2), '\xC0')
     CMPPD_xxi = xmminsn('\x66', rex_nw, '\x0F\xC2', register(1,8), register(2), '\xC0', immediate(3, 'b'))
     CMPPS_xxi = xmminsn(        rex_nw, '\x0F\xC2', register(1,8), register(2), '\xC0', immediate(3, 'b'))
 
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -10,7 +10,7 @@
     xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
     X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, AddressLoc)
 from rpython.jit.backend.llsupport.vector_ext import VectorExt
-from rpython.jit.backend.llsupport.regalloc import get_scale
+from rpython.jit.backend.llsupport.regalloc import get_scale, TempVar
 from rpython.jit.metainterp.resoperation import (rop, ResOperation,
         VectorOp, VectorGuardOp)
 from rpython.rlib.objectmodel import we_are_translated, always_inline
@@ -33,6 +33,14 @@
     raise NotImplementedError(msg)
 # DUP END
 
+class TempVector(TempVar):
+    def __init__(self, type):
+        self.type = type
+    def is_vector(self):
+        return True
+    def __repr__(self):
+        return "<TempVector At %s>" % (id(self),)
+
 class X86VectorExt(VectorExt):
     def setup_once(self, asm):
         if detect_feature.detect_sse4_1():
@@ -292,29 +300,50 @@
             self.mc.XORPD(src, heap(self.float_const_neg_addr))
 
     def genop_vec_float_eq(self, op, arglocs, resloc):
-        _, rhsloc, sizeloc = arglocs
+        lhsloc, rhsloc, sizeloc = arglocs
         size = sizeloc.value
         if size == 4:
-            self.mc.CMPPS_xxi(resloc.value, rhsloc.value, 0) # 0 means equal
+            self.mc.CMPPS_xxi(lhsloc.value, rhsloc.value, 0) # 0 means equal
         else:
-            self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 0)
+            self.mc.CMPPD_xxi(lhsloc.value, rhsloc.value, 0)
+        self.flush_vec_cc(rx86.Conditions["E"], lhsloc, resloc, sizeloc.value)
+
+    def flush_vec_cc(self, rev_cond, lhsloc, resloc, size):
+        # After emitting an instruction that leaves a boolean result in
+        # a condition code (cc), call this.  In the common case, result_loc
+        # will be set to SPP by the regalloc, which in this case means
+        # "propagate it between this operation and the next guard by keeping
+        # it in the cc".  In the uncommon case, result_loc is another
+        # register, and we emit a load from the cc into this register.
+
+        if resloc is ebp:
+            self.guard_success_cc = condition
+        else:
+            assert lhsloc is xmm0
+            maskloc = X86_64_XMM_SCRATCH_REG
+            self.mc.MOVAPD(maskloc, heap(self.element_ones[get_scale(size)]))
+            self.mc.PXOR(resloc, resloc)
+            # note that xmm0 contains true false for each element by the last compare operation
+            self.mc.PBLENDVB_xx(resloc.value, maskloc.value)
 
     def genop_vec_float_ne(self, op, arglocs, resloc):
-        _, rhsloc, sizeloc = arglocs
+        lhsloc, rhsloc, sizeloc = arglocs
         size = sizeloc.value
         # b(100) == 1 << 2 means not equal
         if size == 4:
-            self.mc.CMPPS_xxi(resloc.value, rhsloc.value, 1 << 2)
+            self.mc.CMPPS_xxi(lhsloc.value, rhsloc.value, 1 << 2)
         else:
-            self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 1 << 2)
+            self.mc.CMPPD_xxi(lhsloc.value, rhsloc.value, 1 << 2)
+        self.flush_vec_cc(rx86.Conditions("NE"), lhsloc, resloc, sizeloc.value)
 
     def genop_vec_int_eq(self, op, arglocs, resloc):
-        _, rhsloc, sizeloc = arglocs
+        lhsloc, rhsloc, sizeloc = arglocs
         size = sizeloc.value
-        self.mc.PCMPEQ(resloc, rhsloc, size)
+        self.mc.PCMPEQ(lhsloc, rhsloc, size)
+        self.flush_vec_cc(rx86.Conditions("E"), lhsloc, resloc, sizeloc.value)
 
     def genop_vec_int_ne(self, op, arglocs, resloc):
-        _, rhsloc, sizeloc = arglocs
+        lhsloc, rhsloc, sizeloc = arglocs
         size = sizeloc.value
         self.mc.PCMPEQ(resloc, rhsloc, size)
         temp = X86_64_XMM_SCRATCH_REG
@@ -325,6 +354,7 @@
         # 11 11 11 11
         # ----------- pxor
         # 00 11 00 00
+        self.flush_vec_cc(rx86.Conditions("NE"), lhsloc, resloc, sizeloc.value)
 
     def genop_vec_int_signext(self, op, arglocs, resloc):
         srcloc, sizeloc, tosizeloc = arglocs
@@ -599,9 +629,55 @@
         lhs = op.getarg(0)
         assert isinstance(lhs, VectorOp)
         args = op.getarglist()
+        # we need to use xmm0
+        lhsloc = self.enforce_var_in_vector_reg(op.getarg(0), args, selected_reg=xmm0)
         rhsloc = self.make_sure_var_in_reg(op.getarg(1), args)
-        lhsloc = self.xrm.force_result_in_reg(op, op.getarg(0), args)
-        self.perform(op, [lhsloc, rhsloc, imm(lhs.bytesize)], lhsloc)
+        resloc = self.force_allocate_vector_reg_or_cc(op)
+        self.perform(op, [lhsloc, rhsloc, imm(lhs.bytesize)], resloc)
+
+    def enforce_var_in_vector_reg(self, arg, forbidden_vars, selected_reg):
+        """ Enforce the allocation in a specific register. This can even be a forbidden
+            register. If it is forbidden, it will be moved to another register.
+            Use with caution, currently this is only used for the vectorization backend
+            instructions.
+        """
+        xrm = self.xrm
+        if selected_reg not in xrm.free_regs:
+            variable = None
+            candidate_to_spill = None
+            for var, reg in self.xrm.reg_bindings.items():
+                if reg is selected_reg:
+                    variable = var
+                else:
+                    if var not in forbidden_vars:
+                        candidate_to_spill = var
+            # do we have a free register?
+            if len(xrm.free_regs) == 0:
+                # spill a non forbidden variable
+                self._spill_var(candidate_to_spill, forbidden_vars, None)
+            loc = xrm.free_regs.pop()
+            self.assembler.mov(selected_reg, loc)
+            reg = xrm.reg_bindings.get(arg, None)
+            if reg:
+                xrm.free_regs.append(reg)
+                self.assembler.mov(reg, selected_reg)
+            xrm.reg_bindings[arg] = selected_reg
+            xrm.reg_bindings[variable] = loc
+
+            return selected_reg
+        return self.make_sure_var_in_reg(arg, forbidden_vars, selected_reg=selected_reg)
+
+    def force_allocate_vector_reg_or_cc(self, var):
+        assert var.type == INT
+        if self.next_op_can_accept_cc(self.operations, self.rm.position):
+            # hack: return the ebp location to mean "lives in CC".  This
+            # ebp will not actually be used, and the location will be freed
+            # after the next op as usual.
+            self.xrm.force_allocate_frame_reg(var)
+            return ebp
+        else:
+            # else, return a regular register (not ebp).
+            return self.xrm.force_allocate_reg(var)
 
     consider_vec_float_ne = consider_vec_float_eq
     consider_vec_int_eq = consider_vec_float_eq

From pypy.commits at gmail.com  Mon Aug  1 13:11:20 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 10:11:20 -0700 (PDT)
Subject: [pypy-commit] pypy jitlog-exact-source-lines: deduplicate
 offset2lineno, moved comment to rpython/tool/error.py (pypyjit)
Message-ID: <579f82b8.8411c20a.bcd4e.cf5b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: jitlog-exact-source-lines
Changeset: r85963:3654fa52664d
Date: 2016-08-01 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/3654fa52664d/

Log:	deduplicate offset2lineno, moved comment to rpython/tool/error.py
	(pypyjit)

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -42,26 +42,11 @@
     from rpython.rlib import rvmprof
     return rvmprof.get_unique_id(bytecode)
 
- at jit.elidable
-def offset2lineno(bytecode, stopat):
-    # see dis.findlinestarts for an explanation. This function is copied from
-    # rpython/tool/error.py
-    # lnotab is a list of [byte inc, line inc, ...]
-    # even position denote byte increments, odd line increments...
-    tab = bytecode.co_lnotab
-    line = bytecode.co_firstlineno
-    addr = 0
-    for i in range(0, len(tab), 2):
-        addr = addr + ord(tab[i])
-        if addr > stopat:
-            break
-        line = line + ord(tab[i+1])
-    return line
-
 @jl.returns(jl.MP_FILENAME, jl.MP_LINENO,
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
+    from pypy.tool.error import offset2lineno
     bcindex = ord(bytecode.co_code[next_instr])
     opname = ""
     if 0 <= bcindex < len(opcode_method_names):
diff --git a/rpython/tool/error.py b/rpython/tool/error.py
--- a/rpython/tool/error.py
+++ b/rpython/tool/error.py
@@ -158,6 +158,8 @@
 
 @jit.elidable
 def offset2lineno(c, stopat):
+    # even position in lnotab denote byte increments, odd line increments.
+    # see dis.findlinestarts in the python std. library for more details
     tab = c.co_lnotab
     line = c.co_firstlineno
     addr = 0

From pypy.commits at gmail.com  Mon Aug  1 13:11:22 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 10:11:22 -0700 (PDT)
Subject: [pypy-commit] pypy jitlog-exact-source-lines: close branch
Message-ID: <579f82ba.4171c20a.640c9.cf32@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: jitlog-exact-source-lines
Changeset: r85964:19af959d833f
Date: 2016-08-01 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/19af959d833f/

Log:	close branch


From pypy.commits at gmail.com  Mon Aug  1 13:11:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 10:11:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: merged branch jitlog-exact-source-lines
Message-ID: <579f82bc.8cc51c0a.f2f11.31a3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r85965:a5b71ba0fa6e
Date: 2016-08-01 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/a5b71ba0fa6e/

Log:	merged branch jitlog-exact-source-lines

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -46,6 +46,7 @@
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
+    from pypy.tool.error import offset2lineno
     bcindex = ord(bytecode.co_code[next_instr])
     opname = ""
     if 0 <= bcindex < len(opcode_method_names):
@@ -53,7 +54,8 @@
     name = bytecode.co_name
     if not name:
         name = ""
-    return (bytecode.co_filename, bytecode.co_firstlineno,
+    line = offset2lineno(bytecode, next_instr)
+    return (bytecode.co_filename, line,
             name, intmask(next_instr), opname)
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
diff --git a/rpython/tool/error.py b/rpython/tool/error.py
--- a/rpython/tool/error.py
+++ b/rpython/tool/error.py
@@ -158,6 +158,8 @@
 
 @jit.elidable
 def offset2lineno(c, stopat):
+    # even position in lnotab denote byte increments, odd line increments.
+    # see dis.findlinestarts in the python std. library for more details
     tab = c.co_lnotab
     line = c.co_firstlineno
     addr = 0

From pypy.commits at gmail.com  Mon Aug  1 13:11:26 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 10:11:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: documented branch
Message-ID: <579f82be.c19d1c0a.a744c.2f5c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r85966:2831d94d2fec
Date: 2016-08-01 19:10 +0200
http://bitbucket.org/pypy/pypy/changeset/2831d94d2fec/

Log:	documented branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -105,3 +105,7 @@
 .. branch: ep2016sprint
 
 Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.

From pypy.commits at gmail.com  Mon Aug  1 13:16:54 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 01 Aug 2016 10:16:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Fix switched None and iterable
 object on stack in GET_AWAITABLE (maybe a better fix is possible)
Message-ID: <579f8406.2916c20a.16ae1.ddf2@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r85967:131f3d3c8682
Date: 2016-08-01 19:16 +0200
http://bitbucket.org/pypy/pypy/changeset/131f3d3c8682/

Log:	Fix switched None and iterable object on stack in GET_AWAITABLE
	(maybe a better fix is possible)

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1441,6 +1441,13 @@
             self.settopvalue(w_iterator)
     
     def GET_AWAITABLE(self, oparg, next_instr):
+        from pypy.objspace.std.noneobject import W_NoneObject
+        if isinstance(self.peekvalue(), W_NoneObject):
+            #switch NoneObject with iterable on stack
+            w_firstnone = self.popvalue()
+            w_i = self.popvalue()
+            self.pushvalue(w_firstnone)
+            self.pushvalue(w_i)
         w_iterable = self.peekvalue()
         w_iter = w_iterable._GetAwaitableIter(self.space)
         self.settopvalue(w_iter)

From pypy.commits at gmail.com  Mon Aug  1 14:45:58 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 01 Aug 2016 11:45:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Merge with py3k
Message-ID: <579f98e6.031dc20a.f2c35.f616@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r85968:2688a6d60ce4
Date: 2016-08-01 20:41 +0200
http://bitbucket.org/pypy/pypy/changeset/2688a6d60ce4/

Log:	Merge with py3k

diff too long, truncating to 2000 out of 43939 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -85,10 +85,11 @@
     pass
 
 def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')):
+    import ast
     result = []
     for word in re_word.findall(line):
         if word.startswith('"'):
-            word = eval(word)
+            word = ast.literal_eval(word)
         result.append(word)
     return result
 
diff --git a/lib-python/2.7/test/test_hash.py b/lib-python/2.7/test/test_hash.py
--- a/lib-python/2.7/test/test_hash.py
+++ b/lib-python/2.7/test/test_hash.py
@@ -174,7 +174,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 
diff --git a/lib-python/3/test/test_unicode.py b/lib-python/3/test/test_unicode.py
--- a/lib-python/3/test/test_unicode.py
+++ b/lib-python/3/test/test_unicode.py
@@ -2604,7 +2604,8 @@
     def test_getnewargs(self):
         text = 'abc'
         args = text.__getnewargs__()
-        self.assertIsNot(args[0], text)
+        if support.check_impl_detail():
+            self.assertIsNot(args[0], text)
         self.assertEqual(args[0], text)
         self.assertEqual(len(args), 1)
 
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -418,7 +418,7 @@
     RegrTest('test_threading.py', usemodules="thread", core=True),
     RegrTest('test_threading_local.py', usemodules="thread", core=True),
     RegrTest('test_threadsignals.py', usemodules="thread"),
-    RegrTest('test_time.py', core=True, usemodules="struct"),
+    RegrTest('test_time.py', core=True, usemodules="struct thread _rawffi"),
     RegrTest('test_timeit.py'),
     RegrTest('test_timeout.py'),
     RegrTest('test_tk.py'),
@@ -452,7 +452,7 @@
     RegrTest('test_userstring.py', core=True),
     RegrTest('test_uu.py'),
     RegrTest('test_uuid.py'),
-    RegrTest('test_venv.py'),
+    RegrTest('test_venv.py', usemodules="struct"),
     RegrTest('test_wait3.py', usemodules="thread"),
     RegrTest('test_wait4.py', usemodules="thread"),
     RegrTest('test_warnings.py', core=True),
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -198,10 +198,13 @@
     return tp._alignmentofinstances()
 
 @builtinify
-def byref(cdata):
+def byref(cdata, offset=0):
     # "pointer" is imported at the end of this module to avoid circular
     # imports
-    return pointer(cdata)
+    ptr = pointer(cdata)
+    if offset != 0:
+        ptr._buffer[0] += offset
+    return ptr
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -0,0 +1,91 @@
+# Note: uses the CFFI out-of-line ABI mode.  We can't use the API
+# mode because ffi.compile() needs to run the compiler, which
+# needs 'subprocess', which needs 'msvcrt' and '_subprocess',
+# which depend on '_pypy_winbase_cffi' already.
+#
+# Note that if you need to regenerate _pypy_winbase_cffi and
+# can't use a preexisting PyPy to do that, then running this
+# file should work as long as 'subprocess' is not imported
+# by cffi.  I had to hack in 'cffi._pycparser' to move an
+#'import subprocess' to the inside of a function.  (Also,
+# CPython+CFFI should work as well.)
+#
+# This module supports both msvcrt.py and _subprocess.py.
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pypy_winbase_cffi", None)
+
+# ---------- MSVCRT ----------
+
+ffi.cdef("""
+typedef unsigned short wint_t;
+
+int _open_osfhandle(intptr_t osfhandle, int flags);
+intptr_t _get_osfhandle(int fd);
+int _setmode(int fd, int mode);
+int _locking(int fd, int mode, long nbytes);
+
+int _kbhit(void);
+int _getch(void);
+wint_t _getwch(void);
+int _getche(void);
+wint_t _getwche(void);
+int _putch(int);
+wint_t _putwch(wchar_t);
+int _ungetch(int);
+wint_t _ungetwch(wint_t);
+""")
+
+# ---------- SUBPROCESS ----------
+
+ffi.cdef("""
+typedef struct {
+    DWORD  cb;
+    char * lpReserved;
+    char * lpDesktop;
+    char * lpTitle;
+    DWORD  dwX;
+    DWORD  dwY;
+    DWORD  dwXSize;
+    DWORD  dwYSize;
+    DWORD  dwXCountChars;
+    DWORD  dwYCountChars;
+    DWORD  dwFillAttribute;
+    DWORD  dwFlags;
+    WORD   wShowWindow;
+    WORD   cbReserved2;
+    LPBYTE lpReserved2;
+    HANDLE hStdInput;
+    HANDLE hStdOutput;
+    HANDLE hStdError;
+} STARTUPINFO, *LPSTARTUPINFO;
+
+typedef struct {
+    HANDLE hProcess;
+    HANDLE hThread;
+    DWORD  dwProcessId;
+    DWORD  dwThreadId;
+} PROCESS_INFORMATION, *LPPROCESS_INFORMATION;
+
+DWORD WINAPI GetVersion(void);
+BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD);
+BOOL WINAPI CloseHandle(HANDLE);
+HANDLE WINAPI GetCurrentProcess(void);
+BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
+                            DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
+BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
+BOOL WINAPI TerminateProcess(HANDLE, UINT);
+HANDLE WINAPI GetStdHandle(DWORD);
+""")
+
+# --------------------
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -0,0 +1,10 @@
+# auto-generated file
+import _cffi_backend
+
+ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
+    _version = 0x2601,
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+)
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -10,152 +10,99 @@
 
 # Declare external Win32 functions
 
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
+from _pypy_winbase_cffi import ffi as _ffi
+_kernel32 = _ffi.dlopen('kernel32')
 
 GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
 
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
 
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
+# Now the _subprocess module implementation
 
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
+def _WinError():
+    code, message = _ffi.getwinerror()
+    raise WindowsError(code, message)
 
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
+_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
 
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-_GetModuleFileNameW = _kernel32.GetModuleFileNameW
-_GetModuleFileNameW.argtypes = [ctypes.c_int, ctypes.c_wchar_p, ctypes.c_uint]
-_GetModuleFileNameW.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessW
-_CreateProcess.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_wchar_p, ctypes.c_wchar_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
-
-# Now the _winapi module implementation
-
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
-
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
+class _handle(object):
+    def __init__(self, c_handle):
+        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
+        self.c_handle = c_handle
+        if int(self) != -1:
+            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
 
     def __int__(self):
-        return self.handle
+        return int(_ffi.cast("intptr_t", self.c_handle))
 
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
+    def __repr__(self):
+        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
 
     def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
+        h = int(self)
+        if h != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+        return h
 
     def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
+        if int(self) != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+            _kernel32.CloseHandle(c_handle)
 
 def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
+    handles = _ffi.new("HANDLE[2]")
 
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
+    res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size)
 
     if not res:
         raise _WinError()
 
-    return _handle(read.value), _handle(write.value)
+    return _handle(handles[0]), _handle(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
+    return _handle(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
+    # CPython: the first three arguments are expected to be integers
+    target = _ffi.new("HANDLE[1]")
 
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
+    res = _kernel32.DuplicateHandle(
+        _ffi.cast("HANDLE", source_process),
+        _ffi.cast("HANDLE", source),
+        _ffi.cast("HANDLE", target_process),
+        target, access, inherit, options)
 
     if not res:
         raise _WinError()
 
-    return _handle(target.value)
+    return _handle(target[0])
+
+def _z(input):
+    if input is None:
+        return _ffi.NULL
+    if isinstance(input, basestring):
+        return str(input)
+    raise TypeError("string/unicode/None expected, got %r" % (
+        type(input).__name__,))
 
 def CreateProcess(name, command_line, process_attr, thread_attr,
                   inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
+    si = _ffi.new("STARTUPINFO *")
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
+        # CPython: these three handles are expected to be _handle objects
         if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
+            si.hStdInput = startup_info.hStdInput.c_handle
         if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
+            si.hStdOutput = startup_info.hStdOutput.c_handle
         if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
+            si.hStdError = startup_info.hStdError.c_handle
 
-    pi = _PROCESS_INFORMATION()
+    pi = _ffi.new("PROCESS_INFORMATION *")
     flags |= CREATE_UNICODE_ENVIRONMENT
 
     if env is not None:
@@ -164,47 +111,55 @@
             envbuf += "%s=%s\0" % (k, v)
         envbuf += '\0'
     else:
-        envbuf = None
+        envbuf = _ffi.NULL
 
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
+    res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL,
+                                   _ffi.NULL, inherit, flags, envbuf,
+                                   _z(start_dir), si, pi)
 
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
 
 def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
+    # CPython: the first argument is expected to be an integer.
+    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
+                                        milliseconds)
     if res < 0:
         raise _WinError()
 
     return res
 
 def GetExitCodeProcess(handle):
-    code = _c_int()
+    # CPython: the first argument is expected to be an integer.
+    code = _ffi.new("DWORD[1]")
 
-    res = _GetExitCodeProcess(int(handle), _byref(code))
+    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
 
     if not res:
         raise _WinError()
 
-    return code.value
+    return code[0]
 
 def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
+    # CPython: the first argument is expected to be an integer.
+    # The second argument is silently wrapped in a UINT.
+    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+                                     _ffi.cast("UINT", exitcode))
 
     if not res:
         raise _WinError()
 
 def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
+    stdhandle = _ffi.cast("DWORD", stdhandle)
+    res = _kernel32.GetStdHandle(stdhandle)
 
     if not res:
         return None
     else:
-        return res
+        # note: returns integer, not handle object
+        return int(_ffi.cast("intptr_t", res))
 
 def CloseHandle(handle):
     res = _CloseHandle(handle)
diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py
--- a/lib_pypy/cffi/_pycparser/__init__.py
+++ b/lib_pypy/cffi/_pycparser/__init__.py
@@ -10,7 +10,6 @@
 __all__ = ['c_lexer', 'c_parser', 'c_ast']
 __version__ = '2.14'
 
-from subprocess import Popen, PIPE
 from .c_parser import CParser
 
 
@@ -28,6 +27,7 @@
         When successful, returns the preprocessed file's contents.
         Errors from cpp will be printed out.
     """
+    from subprocess import Popen, PIPE
     path_list = [cpp_path]
     if isinstance(cpp_args, list):
         path_list += cpp_args
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -2,7 +2,7 @@
 import __pypy__
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -7,26 +7,39 @@
 # XXX incomplete: implemented only functions needed by subprocess.py
 # PAC: 2010/08 added MS locking for Whoosh
 
-import ctypes
+# 07/2016: rewrote in CFFI
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The 'msvcrt' module is only available on Windows")
+
+import _rawffi
+from _pypy_winbase_cffi import ffi as _ffi
+_lib = _ffi.dlopen(_rawffi.get_libc().name)
+
 import errno
-from ctypes_support import standard_c_lib as _c
-from ctypes_support import get_errno
-
-try:
-    open_osfhandle = _c._open_osfhandle
-except AttributeError: # we are not on windows
-    raise ImportError
 
 try: from __pypy__ import builtinify, validate_fd
 except ImportError: builtinify = validate_fd = lambda f: f
 
 
-open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int]
-open_osfhandle.restype = ctypes.c_int
+def _ioerr():
+    e = _ffi.errno
+    raise IOError(e, errno.errorcode[e])
 
-_get_osfhandle = _c._get_osfhandle
-_get_osfhandle.argtypes = [ctypes.c_int]
-_get_osfhandle.restype = ctypes.c_int
+
+ at builtinify
+def open_osfhandle(fd, flags):
+    """"open_osfhandle(handle, flags) -> file descriptor
+
+    Create a C runtime file descriptor from the file handle handle. The
+    flags parameter should be a bitwise OR of os.O_APPEND, os.O_RDONLY,
+    and os.O_TEXT. The returned file descriptor may be used as a parameter
+    to os.fdopen() to create a file object."""
+    fd = _lib._open_osfhandle(fd, flags)
+    if fd == -1:
+        _ioerr()
+    return fd
 
 @builtinify
 def get_osfhandle(fd):
@@ -38,62 +51,74 @@
         validate_fd(fd)
     except OSError as e:
         raise IOError(*e.args)
-    return _get_osfhandle(fd)
+    result = _lib._get_osfhandle(fd)
+    if result == -1:
+        _ioerr()
+    return result
 
-setmode = _c._setmode
-setmode.argtypes = [ctypes.c_int, ctypes.c_int]
-setmode.restype = ctypes.c_int
+ at builtinify
+def setmode(fd, flags):
+    """setmode(fd, mode) -> Previous mode
+
+    Set the line-end translation mode for the file descriptor fd. To set
+    it to text mode, flags should be os.O_TEXT; for binary, it should be
+    os.O_BINARY."""
+    flags = _lib._setmode(fd, flags)
+    if flags == -1:
+        _ioerr()
+    return flags
 
 LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5)
 
-_locking = _c._locking
-_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_locking.restype = ctypes.c_int
-
 @builtinify
 def locking(fd, mode, nbytes):
-    '''lock or unlock a number of bytes in a file.'''
-    rv = _locking(fd, mode, nbytes)
+    """"locking(fd, mode, nbytes) -> None
+
+    Lock part of a file based on file descriptor fd from the C runtime.
+    Raises IOError on failure. The locked region of the file extends from
+    the current file position for nbytes bytes, and may continue beyond
+    the end of the file. mode must be one of the LK_* constants listed
+    below. Multiple regions in a file may be locked at the same time, but
+    may not overlap. Adjacent regions are not merged; they must be unlocked
+    individually."""
+    rv = _lib._locking(fd, mode, nbytes)
     if rv != 0:
-        e = get_errno()
-        raise IOError(e, errno.errorcode[e])
+        _ioerr()
 
 # Console I/O routines
 
-kbhit = _c._kbhit
-kbhit.argtypes = []
-kbhit.restype = ctypes.c_int
+kbhit = _lib._kbhit
 
-getch = _c._getch
-getch.argtypes = []
-getch.restype = ctypes.c_char
+ at builtinify
+def getch():
+    return chr(_lib._getch())
 
-getwch = _c._getwch
-getwch.argtypes = []
-getwch.restype = ctypes.c_wchar
+ at builtinify
+def getwch():
+    return unichr(_lib._getwch())
 
-getche = _c._getche
-getche.argtypes = []
-getche.restype = ctypes.c_char
+ at builtinify
+def getche():
+    return chr(_lib._getche())
 
-getwche = _c._getwche
-getwche.argtypes = []
-getwche.restype = ctypes.c_wchar
+ at builtinify
+def getwche():
+    return unichr(_lib._getwche())
 
-putch = _c._putch
-putch.argtypes = [ctypes.c_char]
-putch.restype = None
+ at builtinify
+def putch(ch):
+    _lib._putch(ord(ch))
 
-putwch = _c._putwch
-putwch.argtypes = [ctypes.c_wchar]
-putwch.restype = None
+ at builtinify
+def putwch(ch):
+    _lib._putwch(ord(ch))
 
-ungetch = _c._ungetch
-ungetch.argtypes = [ctypes.c_char]
-ungetch.restype = None
+ at builtinify
+def ungetch(ch):
+    if _lib._ungetch(ord(ch)) == -1:   # EOF
+        _ioerr()
 
-ungetwch = _c._ungetwch
-ungetwch.argtypes = [ctypes.c_wchar]
-ungetwch.restype = None
-
-del ctypes
+ at builtinify
+def ungetwch(ch):
+    if _lib._ungetwch(ord(ch)) == -1:   # EOF
+        _ioerr()
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -40,7 +40,7 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
     "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy"
-    "faulthandler",
+    "faulthandler", "_jitlog",
 ])
 
 from rpython.jit.backend import detect_cpu
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -94,6 +94,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -128,9 +142,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -138,11 +149,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -171,28 +178,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
 
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
-
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -104,27 +104,24 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev liblzma-dev
-
-For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+    tk-dev libgc-dev \
+    liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel
-
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+    gdbm-devel \
+    xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses
+    libexpat-devel libffi-devel python-curses \
+    xz-devel  # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
-
 On Mac OS X, most of these build-time dependencies are installed alongside
 the Developer Tools. However, note that in order for the installation to
 find them you may need to run::
diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt
--- a/pypy/doc/config/commandline.txt
+++ b/pypy/doc/config/commandline.txt
@@ -9,7 +9,7 @@
 PyPy Python interpreter options
 -------------------------------
 
-The following options can be used after ``translate.py
+The following options can be used after ``rpython
 targetpypystandalone`` or as options to ``py.py``.
 
 .. GENERATE: objspace
@@ -22,7 +22,7 @@
 General translation options
 ---------------------------
 
-The following are options of ``translate.py``.  They must be
+The following are options of ``bin/rpython``.  They must be
 given before the ``targetxxx`` on the command line.
 
 * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]`
diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst
--- a/pypy/doc/config/index.rst
+++ b/pypy/doc/config/index.rst
@@ -15,12 +15,12 @@
 
     ./py.py <`objspace options`_>
 
-and the ``translate.py`` translation entry
+and the ``rpython/bin/rpython`` translation entry
 point which takes arguments of this form:
 
 .. parsed-literal::
 
-    ./translate.py <`translation options`_> <target>
+    ./rpython/bin/rpython <`translation options`_> <target>
 
 For the common case of ``<target>`` being ``targetpypystandalone.py``,
 you can then pass the `object space options`_ after
@@ -28,7 +28,7 @@
 
 .. parsed-literal::
 
-    ./translate.py <`translation options`_> targetpypystandalone.py <`objspace options`_>
+    ./rpython/bin/rpython <`translation options`_> targetpypystandalone.py <`objspace options`_>
 
 There is an `overview`_ of all command line arguments that can be
 passed in either position.
diff --git a/pypy/doc/config/opt.rst b/pypy/doc/config/opt.rst
--- a/pypy/doc/config/opt.rst
+++ b/pypy/doc/config/opt.rst
@@ -4,8 +4,8 @@
 This meta-option selects a default set of optimization
 settings to use during a translation.  Usage::
 
-    translate.py --opt=#
-    translate.py -O#
+    bin/rpython --opt=#
+    bin/rpython -O#
 
 where ``#`` is the desired optimization level.  The valid choices are:
 
diff --git a/pypy/doc/config/translation.dont_write_c_files.txt b/pypy/doc/config/translation.dont_write_c_files.txt
--- a/pypy/doc/config/translation.dont_write_c_files.txt
+++ b/pypy/doc/config/translation.dont_write_c_files.txt
@@ -1,4 +1,4 @@
 write the generated C files to ``/dev/null`` instead of to the disk. Useful if
-you want to use translate.py as a benchmark and don't want to access the disk.
+you want to use translation as a benchmark and don't want to access the disk.
 
 .. _`translation documentation`: ../translation.html
diff --git a/pypy/doc/config/translation.fork_before.txt b/pypy/doc/config/translation.fork_before.txt
--- a/pypy/doc/config/translation.fork_before.txt
+++ b/pypy/doc/config/translation.fork_before.txt
@@ -1,4 +1,4 @@
 This is an option mostly useful when working on the PyPy toolchain. If you use
-it, translate.py will fork before the specified phase. If the translation
+it, translation will fork before the specified phase. If the translation
 crashes after that fork, you can fix the bug in the toolchain, and continue
 translation at the fork-point.
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -122,7 +122,7 @@
     $ hg up reflex-support         # optional
 
     # This example shows python, but using pypy-c is faster and uses less memory
-    $ python rpython/translator/goal/translate.py --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy
+    $ python rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy
 
 This will build a ``pypy-c`` that includes the cppyy module, and through that,
 Reflex support.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -315,13 +315,28 @@
 
  - ``complex``
 
+ - ``str`` (empty or single-character strings only)
+
+ - ``unicode`` (empty or single-character strings only)
+
+ - ``tuple`` (empty tuples only)
+
+ - ``frozenset`` (empty frozenset only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
-Notably missing from the list above are ``str`` and ``unicode``.  If your
-code relies on comparing strings with ``is``, then it might break in PyPy.
+Note that strings of length 2 or greater can be equal without being
+identical.  Similarly, ``x is (2,)`` is not necessarily true even if
+``x`` contains a tuple and ``x == (2,)``.  The uniqueness rules apply
+only to the particular cases described above.  The ``str``, ``unicode``,
+``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a
+test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was
+equal to ``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is
+closer to CPython's, which caches precisely the empty tuple/frozenset,
+and (generally but not always) the strings and unicodes of length <= 1.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -335,3 +335,60 @@
 
 This will disable SELinux's protection and allow PyPy to configure correctly.
 Be sure to enable it again if you need it!
+
+
+How should I report a bug?
+--------------------------
+
+Our bug tracker is here: https://bitbucket.org/pypy/pypy/issues/
+
+Missing features or incompatibilities with CPython are considered
+bugs, and they are welcome.  (See also our list of `known
+incompatibilities`__.)
+
+.. __: http://pypy.org/compat.html
+
+For bugs of the kind "I'm getting a PyPy crash or a strange
+exception", please note that: **We can't do anything without
+reproducing the bug ourselves**.  We cannot do anything with
+tracebacks from gdb, or core dumps.  This is not only because the
+standard PyPy is compiled without debug symbols.  The real reason is
+that a C-level traceback is usually of no help at all in PyPy.
+Debugging PyPy can be annoying.
+
+In more details:
+
+* First, please give the exact PyPy version, and the OS.
+
+* It might help focus our search if we know if the bug can be
+  reproduced on a "``pypy --jit off``" or not.  If "``pypy --jit
+  off``" always works, then the problem might be in the JIT.
+  Otherwise, we know we can ignore that part.
+
+* If you got the bug using only Open Source components, please give a
+  step-by-step guide that we can follow to reproduce the problem
+  ourselves.  Don't assume we know anything about any program other
+  than PyPy.  We would like a guide that we can follow point by point
+  (without guessing or having to figure things out)
+  on a machine similar to yours, starting from a bare PyPy, until we
+  see the same problem.  (If you can, you can try to reduce the number
+  of steps and the time it needs to run, but that is not mandatory.)
+
+* If the bug involves Closed Source components, or just too many Open
+  Source components to install them all ourselves, then maybe you can
+  give us some temporary ssh access to a machine where the bug can be
+  reproduced.  Or, maybe we can download a VirtualBox or VMWare
+  virtual machine where the problem occurs.
+
+* If giving us access would require us to use tools other than ssh,
+  make appointments, or sign a NDA, then we can consider a commerical
+  support contract for a small sum of money.
+
+* If even that is not possible for you, then sorry, we can't help.
+
+Of course, you can try to debug the problem yourself, and we can help
+you get started if you ask on the #pypy IRC channel, but be prepared:
+debugging an annoying PyPy problem usually involves quite a lot of gdb
+in auto-generated C code, and at least some knowledge about the
+various components involved, from PyPy's own RPython source code to
+the GC and possibly the JIT.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -39,17 +39,16 @@
 library.
 
 If you want to install 3rd party libraries, the most convenient way is
-to install pip_ (unless you want to install virtualenv as explained
-below; then you can directly use pip inside virtualenvs):
+to install pip_ using ensurepip_ (unless you want to install virtualenv as 
+explained below; then you can directly use pip inside virtualenvs):
 
 .. code-block:: console
 
-    $ curl -O https://bootstrap.pypa.io/get-pip.py
-    $ ./pypy-2.1/bin/pypy get-pip.py
-    $ ./pypy-2.1/bin/pip install pygments  # for example
+    $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install pygments  # for example
 
-Third party libraries will be installed in ``pypy-2.1/site-packages``, and
-the scripts in ``pypy-2.1/bin``.
+Third party libraries will be installed in ``pypy-xxx/site-packages``, and
+the scripts in ``pypy-xxx/bin``.
 
 
 Installing using virtualenv
@@ -61,7 +60,7 @@
 checkout::
 
 	# from a tarball
-	$ virtualenv -p /opt/pypy-c-jit-41718-3fb486695f20-linux/bin/pypy my-pypy-env
+	$ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env
 
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
@@ -69,7 +68,7 @@
 Note that bin/python is now a symlink to bin/pypy.
 
 .. _pip: http://pypi.python.org/pypi/pip
-
+.. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html
 
 Building PyPy yourself
 ~~~~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -0,0 +1,41 @@
+==========
+PyPy 5.3.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.3.0, released last week,
+due to issues_ reported by users.
+
+Thanks to those who reported the issues.
+
+.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,13 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
 .. branch: fix-gen-dfa
 
 Resolves an issue with the generator script to build the dfa for Python syntax.
@@ -19,3 +26,82 @@
 .. branch: s390x-5.3-catchup
 
 Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -0,0 +1,15 @@
+===========================
+What's new in PyPy2.7 5.3.1
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3.0
+.. startrev: f4d726d1a010
+
+
+A bug-fix release, merging these changes:
+
+  * Add include guards to pymem.h, fixes issue #2321
+
+  * Make vmprof build on OpenBSD, from pull request #456
+
+  * Fix ``bytearray('').replace('a', 'ab')``, issue #2324
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -416,28 +416,12 @@
                        try_node.get_lineno(), try_node.get_column())
 
     def handle_with_stmt(self, with_node, is_async):
-        body = self.handle_suite(with_node.get_child(-1))
-        i = with_node.num_children() - 1
-        while True:
-            i -= 2
-            item = with_node.get_child(i)
-            test = self.handle_expr(item.get_child(0))
-            if item.num_children() == 3:
-                target = self.handle_expr(item.get_child(2))
-                self.set_context(target, ast.Store)
-            else:
-                target = None
             if is_async:
                 wi = ast.AsyncWith(test, target, body, with_node.get_lineno(),
                               with_node.get_column())
             else:
                 wi = ast.With(test, target, body, with_node.get_lineno(),
                               with_node.get_column())
-            if i == 1:
-                break
-            body = [wi]
-        return wi
-
     def handle_with_item(self, item_node):
         test = self.handle_expr(item_node.get_child(0))
         if item_node.num_children() == 3:
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1129,7 +1129,7 @@
         assert space.eq_w(s.s, space.wrap("hi implicitly extra"))
         s = self.get_first_expr("b'hi' b' implicitly' b' extra'")
         assert isinstance(s, ast.Bytes)
-        assert space.eq_w(s.s, space.wrapbytes("hi implicitly extra"))
+        assert space.eq_w(s.s, space.newbytes("hi implicitly extra"))
         raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
         sentence = u"Die M&#228;nner &#228;rgen sich!"
         source = u"# coding: utf-7\nstuff = '%s'" % (sentence,)
@@ -1184,7 +1184,7 @@
         s = ast_from_node(space, tree, info).body[0].value
         assert isinstance(s, ast.Str)
         assert space.eq_w(s.s, space.wrap(u'&#199;'))
- 
+
     def test_string_bug(self):
         space = self.space
         source = '# -*- encoding: utf8 -*-\nstuff = "x \xc3\xa9 \\n"\n'
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -965,7 +965,20 @@
         """
         self.simple_test(source, 'ok', 1)
 
-    def test_remove_docstring(self):
+    @py.test.mark.parametrize('expr, result', [
+        ("f1.__doc__", None),
+        ("f2.__doc__", 'docstring'),
+        ("f2()", 'docstring'),
+        ("f3.__doc__", None),
+        ("f3()", 'bar'),
+        ("C1.__doc__", None),
+        ("C2.__doc__", 'docstring'),
+        ("C3.field", 'not docstring'),
+        ("C4.field", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("__doc__", None),])
+    def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
             'docstring'
@@ -989,19 +1002,7 @@
         code_w.remove_docstrings(self.space)
         dict_w = self.space.newdict();
         code_w.exec_code(self.space, dict_w, dict_w)
-
-        yield self.check, dict_w, "f1.__doc__", None
-        yield self.check, dict_w, "f2.__doc__", 'docstring'
-        yield self.check, dict_w, "f2()", 'docstring'
-        yield self.check, dict_w, "f3.__doc__", None
-        yield self.check, dict_w, "f3()", 'bar'
-        yield self.check, dict_w, "C1.__doc__", None
-        yield self.check, dict_w, "C2.__doc__", 'docstring'
-        yield self.check, dict_w, "C3.field", 'not docstring'
-        yield self.check, dict_w, "C4.field", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "__doc__", None
+        self.check(dict_w, expr, result)
 
     def test_assert_skipping(self):
         space = self.space
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -422,7 +422,7 @@
     if not (space.isinstance_w(w_obj, space.w_str) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
-                   "AST string must be of type str or unicode")
+                    "AST string must be of type str or unicode")
     return w_obj
 
 def get_field(space, w_node, name, optional):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1056,7 +1056,7 @@
         return (None, None)
 
     def newlist_bytes(self, list_s):
-        return self.newlist([self.wrapbytes(s) for s in list_s])
+        return self.newlist([self.newbytes(s) for s in list_s])
 
     def newlist_unicode(self, list_u):
         return self.newlist([self.wrap(u) for u in list_u])
@@ -1072,7 +1072,7 @@
         return make_empty_list_with_size(self, sizehint)
 
     def wrap_fsdecoded(self, x):
-        return self.fsdecode(self.wrapbytes(x))
+        return self.fsdecode(self.newbytes(x))
 
     @jit.unroll_safe
     def exception_match(self, w_exc_type, w_check_class):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -958,7 +958,7 @@
             elif name != '__args__' and name != 'args_w':
                 spec = unwrap_spec[i]
                 if isinstance(defaultval, str) and spec not in [str]:
-                    defs_w.append(space.wrapbytes(defaultval))
+                    defs_w.append(space.newbytes(defaultval))
                 else:
                     defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
@@ -978,7 +978,7 @@
                 if isinstance(spec, WrappedDefault):
                     default_value = spec.default_value
                     if isinstance(default_value, str):
-                        w_default = space.wrapbytes(default_value)
+                        w_default = space.newbytes(default_value)
                     else:
                         w_default = space.wrap(default_value)
                     assert isinstance(w_default, W_Root)
diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -19,7 +19,7 @@
 def compilecode(space, source, filename, cmd='exec'):
     w = space.wrap
     w_code = space.builtin.call(
-        'compile', space.wrapbytes(source), space.wrap_fsdecoded(filename),
+        'compile', space.newbytes(source), space.wrap_fsdecoded(filename),
         w(cmd), w(0), w(0))
     pycode = space.interp_w(eval.Code, w_code)
     return pycode
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -28,6 +28,7 @@
 def unpack_str_tuple(space,w_str_tuple):
     return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
 
+
 # Magic numbers for the bytecode version in code objects.
 # See comments in pypy/module/imp/importing.
 cpython_magic, = struct.unpack("<i", imp.get_magic())   # host magic number
@@ -420,14 +421,14 @@
             w(self.co_nlocals),
             w(self.co_stacksize),
             w(self.co_flags),
-            space.wrapbytes(self.co_code),
+            space.newbytes(self.co_code),
             space.newtuple(self.co_consts_w),
             space.newtuple(self.co_names_w),
             space.newtuple([w(v) for v in self.co_varnames]),
             w(self.co_filename),
             w(self.co_name),
             w(self.co_firstlineno),
-            space.wrapbytes(self.co_lnotab),
+            space.newbytes(self.co_lnotab),
             space.newtuple([w(v) for v in self.co_freevars]),
             space.newtuple([w(v) for v in self.co_cellvars]),
             w(self.magic),
@@ -448,7 +449,7 @@
         space = self.space
         # co_name should be an identifier
         name = self.co_name.decode('utf-8')
-        fn = space.fsdecode_w(space.wrapbytes(self.co_filename))
+        fn = space.fsdecode_w(space.newbytes(self.co_filename))
         return space.wrap(u'<code object %s at 0x%s, file "%s", line %d>' % (
             name, unicode(self.getaddrstring(space)), fn,
             -1 if self.co_firstlineno == 0 else self.co_firstlineno))
diff --git a/pypy/interpreter/pyparser/automata.py b/pypy/interpreter/pyparser/automata.py
--- a/pypy/interpreter/pyparser/automata.py
+++ b/pypy/interpreter/pyparser/automata.py
@@ -13,12 +13,11 @@
 # PYPY Modification: removed the EMPTY class as it's not needed here
 
 
-# PYPY Modification: we don't need a particuliar DEFAULT class here
-#                    a simple None works fine.
-#                    (Having a DefaultClass inheriting from str makes
-#                     the annotator crash)
-DEFAULT = "\00default" # XXX hack, the rtyper does not support dict of with str|None keys
-                       # anyway using dicts doesn't seem the best final way to store these char indexed tables
+# PYPY Modification: DEFAULT is a singleton, used only in the pre-RPython
+# dicts (see pytokenize.py).  Then DFA.__init__() turns these dicts into
+# more compact strings.
+DEFAULT = object()
+
 # PYPY Modification : removed all automata functions (any, maybe,
 #                     newArcPair, etc.)
 
diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -25,7 +25,7 @@
                                            'replace')
             w_text = space.wrap(text)
         if self.filename is not None:
-            w_filename = space.fsdecode(space.wrapbytes(self.filename))
+            w_filename = space.fsdecode(space.newbytes(self.filename))
         return space.newtuple([space.wrap(self.msg),
                                space.newtuple([w_filename,
                                                space.wrap(self.lineno),
diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -294,7 +294,7 @@
         i = 0
         for k, v in sorted(state.items()):
             i += 1
-            if k == '\x00default':
+            if k == DEFAULT:
                 k = "automata.DEFAULT"
             else:
                 k = repr(k)
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -85,13 +85,13 @@
 
     if rawmode or '\\' not in substr:
         if not unicode_literal:
-            return space.wrapbytes(substr)
+            return space.newbytes(substr)
         else:
             v = unicodehelper.decode_utf8(space, substr)
             return space.wrap(v)
 
     v = PyString_DecodeEscape(space, substr, 'strict', encoding)
-    return space.wrapbytes(v)
+    return space.newbytes(v)
 
 def decode_unicode_utf8(space, s, ps, q):
     # ****The Python 2.7 version, producing UTF-32 escapes****
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -7,7 +7,7 @@
 def recode_to_utf8(space, bytes, encoding):
     if encoding == 'utf-8':
         return bytes
-    w_text = space.call_method(space.wrapbytes(bytes), "decode",
+    w_text = space.call_method(space.newbytes(bytes), "decode",
                                space.wrap(encoding))
     w_recoded = space.call_method(w_text, "encode", space.wrap("utf-8"))
     return space.bytes_w(w_recoded)
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -471,12 +471,12 @@
         space = self.space
         w = space.wrap
         def f(filename):
-            return space.wrapbytes(filename)
+            return space.newbytes(filename)
         app_f = gateway.interp2app_temp(f, unwrap_spec=['fsencode'])
         w_app_f = space.wrap(app_f)
         assert space.eq_w(
             space.call_function(w_app_f, w(u'\udc80')),
-            space.wrapbytes('\x80'))
+            space.newbytes('\x80'))
 
     def test_interp2app_unwrap_spec_typechecks(self):
         from rpython.rlib.rarithmetic import r_longlong
@@ -801,7 +801,7 @@
         w_g = space.wrap(gateway.interp2app_temp(g))
         args = argument.Arguments(space, [])
         w_res = space.call_args(w_g, args)
-        assert space.eq_w(w_res, space.wrapbytes('foo'))
+        assert space.eq_w(w_res, space.newbytes('foo'))
 
 
 class AppTestPyTestMark:
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -345,13 +345,13 @@
 def interp_attrproperty_bytes(name, cls, doc=None):
     "NOT_RPYTHON: initialization-time only"
     def fget(space, obj):
-        return space.wrapbytes(getattr(obj, name))
+        return space.newbytes(getattr(obj, name))
     return GetSetProperty(fget, cls=cls, doc=doc)
 
 def interp_attrproperty_fsdecode(name, cls, doc=None):
     "NOT_RPYTHON: initialization-time only"
     def fget(space, obj):
-        return space.fsdecode(space.wrapbytes(getattr(obj, name)))
+        return space.fsdecode(space.newbytes(getattr(obj, name)))
     return GetSetProperty(fget, cls=cls, doc=doc)
 
 def interp_attrproperty_w(name, cls, doc=None):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -18,7 +18,7 @@
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeDecodeError,
                              space.newtuple([space.wrap(encoding),
-                                             space.wrapbytes(s),
+                                             space.newbytes(s),
                                              space.wrap(startingpos),
                                              space.wrap(endingpos),
                                              space.wrap(msg)]))
@@ -111,7 +111,7 @@
         return space.call_method(w_uni, 'encode',
                                  getfilesystemencoding(space),
                                  space.wrap('surrogateescape'))
-    return space.wrapbytes(bytes)
+    return space.newbytes(bytes)
 
 def encode(space, w_data, encoding=None, errors='strict'):
     from pypy.objspace.std.unicodeobject import encode_object
@@ -141,9 +141,7 @@
     return result
 
 def encode_utf8(space, uni, allow_surrogates=False):
-    # Note that this function never raises UnicodeEncodeError,
-    # since surrogate pairs are allowed.
-    # This is not the case with Python3.
+    # Note that Python3 tends to forbid lone surrogates
     return runicode.unicode_encode_utf_8(
         uni, len(uni), "strict",
         errorhandler=encode_error_handler(space),
diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -43,9 +43,61 @@
             raise       # propagate other errors
         return space.type(w_obj)
 
+
+# ---------- isinstance ----------
+
+
+def p_recursive_isinstance_w(space, w_inst, w_cls):
+    # Copied straight from CPython 2.7.  Does not handle 'cls' being a tuple.
+    if space.isinstance_w(w_cls, space.w_type):
+        return p_recursive_isinstance_type_w(space, w_inst, w_cls)
+
+    check_class(space, w_cls, "isinstance() arg 2 must be a class, type,"
+                              " or tuple of classes and types")
+    try:
+        w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
+    except OperationError as e:
+        if not e.match(space, space.w_AttributeError):
+            raise       # propagate other errors
+        return False
+    else:
+        return p_abstract_issubclass_w(space, w_abstractclass, w_cls)
+
+
+def p_recursive_isinstance_type_w(space, w_inst, w_type):
+    # subfunctionality of p_recursive_isinstance_w(): assumes that w_type is
+    # a type object.  Copied straight from CPython 2.7.
+    if space.isinstance_w(w_inst, w_type):
+        return True
+    try:
+        w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
+    except OperationError as e:
+        if not e.match(space, space.w_AttributeError):
+            raise       # propagate other errors
+    else:
+        if w_abstractclass is not space.type(w_inst):
+            if space.isinstance_w(w_abstractclass, space.w_type):
+                return space.issubtype_w(w_abstractclass, w_type)
+    return False
+
+
 @jit.unroll_safe
 def abstract_isinstance_w(space, w_obj, w_klass_or_tuple, allow_override=False):
     """Implementation for the full 'isinstance(obj, klass_or_tuple)'."""
+    # Copied from CPython 2.7's PyObject_Isinstance().  Additionally,
+    # if 'allow_override' is False (the default), then don't try to
+    # use a custom __instancecheck__ method.
+
+    # WARNING: backward compatibility function name here.  CPython
+    # uses the name "abstract" to refer to the logic of handling
+    # class-like objects, with a "__bases__" attribute.  This function
+    # here is not related to that and implements the full
+    # PyObject_IsInstance() logic.
+
+    # Quick test for an exact match
+    if space.type(w_obj) is w_klass_or_tuple:
+        return True
+
     # -- case (anything, tuple)
     # XXX it might be risky that the JIT sees this
     if space.isinstance_w(w_klass_or_tuple, space.w_tuple):
@@ -55,64 +107,55 @@
         return False
 
     # -- case (anything, type)
-    try:
-        if allow_override:
-            w_result = space.isinstance_allow_override(w_obj, w_klass_or_tuple)
-        else:
-            w_result = space.isinstance(w_obj, w_klass_or_tuple)
-    except OperationError as e:   # if w_klass_or_tuple was not a type, ignore it
-        if not e.match(space, space.w_TypeError):
-            raise       # propagate other errors
-    else:
-        if space.is_true(w_result):
-            return True
-        # From now on we know that w_klass_or_tuple is indeed a type.
-        # Try also to compare it with obj.__class__, if this is not
-        # the same as type(obj).
-        w_pretendtype = abstract_getclass(space, w_obj)
-        try:
-            if space.is_w(w_pretendtype, space.type(w_obj)):
-                return False     # common case: obj.__class__ is type(obj)
-            if not allow_override:
-                return space.issubtype_w(w_pretendtype, w_klass_or_tuple)
-            w_result = space.issubtype_allow_override(w_pretendtype,
-                                                      w_klass_or_tuple)
-        except OperationError as e:
-            if e.async(space):
-                raise
-            return False      # ignore most exceptions
-        else:
-            return space.is_true(w_result)
+    if allow_override:
+        w_check = space.lookup(w_klass_or_tuple, "__instancecheck__")
+        if w_check is not None:
+            # this is the common case: all type objects have a method
+            # __instancecheck__.  The one in the base 'type' type calls
+            # back p_recursive_isinstance_type_w() from the present module.
+            return space.is_true(space.get_and_call_function(
+                w_check, w_klass_or_tuple, w_obj))
 
-    return _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple)
+    return p_recursive_isinstance_w(space, w_obj, w_klass_or_tuple)
 
 
-def _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple):
-    # -- case (anything, abstract-class)
-    check_class(space, w_klass_or_tuple,
-                "isinstance() arg 2 must be a class, type,"
-                " or tuple of classes and types")
-    try:
-        w_abstractclass = space.getattr(w_obj, space.wrap('__class__'))
-    except OperationError as e:
-        if e.async(space):      # ignore most exceptions
-            raise
-        return False
-    else:
-        return _issubclass_recurse(space, w_abstractclass, w_klass_or_tuple)
+# ---------- issubclass ----------
 
 
 @jit.unroll_safe
-def _issubclass_recurse(space, w_derived, w_top):
-    """Internal helper for abstract cases.  Here, w_top cannot be a tuple."""
-    if space.is_w(w_derived, w_top):
-        return True
-    w_bases = _get_bases(space, w_derived)
-    if w_bases is not None:
-        for w_base in space.fixedview(w_bases):
-            if _issubclass_recurse(space, w_base, w_top):
+def p_abstract_issubclass_w(space, w_derived, w_cls):
+    # Copied straight from CPython 2.7, function abstract_issubclass().
+    # Don't confuse this with the function abstract_issubclass_w() below.
+    # Here, w_cls cannot be a tuple.
+    while True:
+        if space.is_w(w_derived, w_cls):
+            return True
+        w_bases = _get_bases(space, w_derived)
+        if w_bases is None:
+            return False
+        bases_w = space.fixedview(w_bases)
+        last_index = len(bases_w) - 1
+        if last_index < 0:
+            return False
+        # Avoid recursivity in the single inheritance case; in general,
+        # don't recurse on the last item in the tuple (loop instead).
+        for i in range(last_index):
+            if p_abstract_issubclass_w(space, bases_w[i], w_cls):
                 return True
-    return False
+        w_derived = bases_w[last_index]
+
+
+def p_recursive_issubclass_w(space, w_derived, w_cls):
+    # From CPython's function of the same name (which as far as I can tell
+    # is not recursive).  Copied straight from CPython 2.7.
+    if (space.isinstance_w(w_cls, space.w_type) and
+        space.isinstance_w(w_derived, space.w_type)):
+        return space.issubtype_w(w_derived, w_cls)
+    #
+    check_class(space, w_derived, "issubclass() arg 1 must be a class")
+    check_class(space, w_cls, "issubclass() arg 2 must be a class"
+                              " or tuple of classes")
+    return p_abstract_issubclass_w(space, w_derived, w_cls)
 
 
 @jit.unroll_safe
@@ -120,32 +163,46 @@
                           allow_override=False):
     """Implementation for the full 'issubclass(derived, klass_or_tuple)'."""
 
-    # -- case (class-like-object, tuple-of-classes)
+    # WARNING: backward compatibility function name here.  CPython
+    # uses the name "abstract" to refer to the logic of handling
+    # class-like objects, with a "__bases__" attribute.  This function
+    # here is not related to that and implements the full
+    # PyObject_IsSubclass() logic.  There is also p_abstract_issubclass_w().
+
+    # -- case (anything, tuple-of-classes)
     if space.isinstance_w(w_klass_or_tuple, space.w_tuple):
         for w_klass in space.fixedview(w_klass_or_tuple):
             if abstract_issubclass_w(space, w_derived, w_klass, allow_override):
                 return True
         return False
 
-    # -- case (type, type)
-    try:
-        if not allow_override:
-            return space.issubtype_w(w_derived, w_klass_or_tuple)
-        w_result = space.issubtype_allow_override(w_derived, w_klass_or_tuple)
-    except OperationError as e:   # if one of the args was not a type, ignore it
-        if not e.match(space, space.w_TypeError):
-            raise       # propagate other errors
-    else:
-        return space.is_true(w_result)
+    # -- case (anything, type)
+    if allow_override:
+        w_check = space.lookup(w_klass_or_tuple, "__subclasscheck__")
+        if w_check is not None:
+            # this is the common case: all type objects have a method
+            # __subclasscheck__.  The one in the base 'type' type calls
+            # back p_recursive_issubclass_w() from the present module.
+            return space.is_true(space.get_and_call_function(
+                w_check, w_klass_or_tuple, w_derived))
 
-    check_class(space, w_derived, "issubclass() arg 1 must be a class")
-    # from here on, we are sure that w_derived is a class-like object
+    return p_recursive_issubclass_w(space, w_derived, w_klass_or_tuple)
 
-    # -- case (class-like-object, abstract-class)
-    check_class(space, w_klass_or_tuple,
-                "issubclass() arg 2 must be a class, type,"
-                " or tuple of classes and types")
-    return _issubclass_recurse(space, w_derived, w_klass_or_tuple)
+
+# ------------------------------------------------------------
+# Exception helpers
+
+def exception_is_valid_obj_as_class_w(space, w_obj):
+    return BaseObjSpace.exception_is_valid_obj_as_class_w(space, w_obj)
+
+def exception_is_valid_class_w(space, w_cls):
+    return BaseObjSpace.exception_is_valid_class_w(space, w_cls)
+
+def exception_getclass(space, w_obj):
+    return BaseObjSpace.exception_getclass(space, w_obj)
+
+def exception_issubclass_w(space, w_cls1, w_cls2):
+    return BaseObjSpace.exception_issubclass_w(space, w_cls1, w_cls2)
 
 # ____________________________________________________________
 # App-level interface
diff --git a/pypy/module/__builtin__/test/test_abstractinst.py b/pypy/module/__builtin__/test/test_abstractinst.py
--- a/pypy/module/__builtin__/test/test_abstractinst.py
+++ b/pypy/module/__builtin__/test/test_abstractinst.py
@@ -226,3 +226,26 @@
         c = C()
         assert isinstance(c, C)
         assert not called
+
+    def test_instancecheck_exception_not_eaten(self):
+        class M(object):
+            def __instancecheck__(self, obj):
+                raise TypeError("foobar")
+
+        e = raises(TypeError, isinstance, 42, M())
+        assert str(e.value) == "foobar"
+
+    def test_issubclass_exception_not_eaten(self):
+        class M(object):
+            def __subclasscheck__(self, subcls):
+                raise TypeError("foobar")
+
+        e = raises(TypeError, issubclass, 42, M())
+        assert str(e.value) == "foobar"
+
+    def test_issubclass_no_fallback(self):
+        class M(object):
+            def __subclasscheck__(self, subcls):
+                return False
+
+        assert issubclass(42, M()) is False
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -40,7 +40,7 @@
             s = self.builder.build()
             self.builder = None
             if strtype is str:
-                return space.wrapbytes(s)
+                return space.newbytes(s)
             else:
                 return space.wrap(s)
 
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,21 +2,10 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod
 from rpython.rlib import jit
 
 
-# XXX maybe temporary: hide llop.int_{floordiv,mod} from the JIT,
-#     because now it expects only Python-style divisions, not the
-#     C-style divisions of these two ll operations
- at jit.dont_look_inside
-def _int_floordiv(n, m):
-    return llop.int_floordiv(lltype.Signed, n, m)
-
- at jit.dont_look_inside
-def _int_mod(n, m):
-    return llop.int_mod(lltype.Signed, n, m)
-
-
 @unwrap_spec(n=int, m=int)
 def int_add(space, n, m):
     return space.wrap(llop.int_add(lltype.Signed, n, m))
@@ -31,11 +20,11 @@
 
 @unwrap_spec(n=int, m=int)
 def int_floordiv(space, n, m):
-    return space.wrap(_int_floordiv(n, m))
+    return space.wrap(int_c_div(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mod(space, n, m):
-    return space.wrap(_int_mod(n, m))
+    return space.wrap(int_c_mod(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_lshift(space, n, m):
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -55,9 +55,9 @@
         start, stop, step, size = space.decode_index4(w_index,
                                                       self.buffer.getlength())
         if step == 0:
-            return space.wrapbytes(self.buffer.getitem(start))
+            return space.newbytes(self.buffer.getitem(start))
         res = self.buffer.getslice(start, stop, step, size)
-        return space.wrapbytes(res)
+        return space.newbytes(res)
 
     def descr_setitem(self, space, w_index, w_newstring):
         start, stop, step, size = space.decode_index4(w_index,
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -220,6 +220,11 @@
         if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK:
             raise oefmt(space.w_SystemError,
                         "libffi failed to build this callback")
+        if closure_ptr.c_user_data != unique_id:
+            raise oefmt(space.w_SystemError,
+                "ffi_prep_closure(): bad user_data (it seems that the "
+                "version of the libffi library seen at runtime is "
+                "different from the 'ffi.h' file seen at compile-time)")
 
     def py_invoke(self, ll_res, ll_args):
         jitdriver1.jit_merge_point(callback=self,
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -84,7 +84,7 @@
         if self.size == 1:
             with cdataobj as ptr:
                 s = ptr[0]
-            return self.space.wrapbytes(s)
+            return self.space.newbytes(s)
         return W_CType.string(self, cdataobj, maxlen)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
@@ -126,7 +126,7 @@
         return self.space.wrap(ord(cdata[0]))
 
     def convert_to_object(self, cdata):
-        return self.space.wrapbytes(cdata[0])
+        return self.space.newbytes(cdata[0])
 
     def _convert_to_char(self, w_ob):
         space = self.space
@@ -146,7 +146,7 @@
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
         s = rffi.charpsize2str(ptr, length)
-        return self.space.wrapbytes(s)
+        return self.space.newbytes(s)
 
 
 # XXX explicitly use an integer type instead of lltype.UniChar here,
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -122,7 +122,7 @@
                         s = rffi.charp2str(ptr)
                     else:
                         s = rffi.charp2strn(ptr, length)
-                    return space.wrapbytes(s)
+                    return space.newbytes(s)
                 #
                 # pointer to a wchar_t: builds and returns a unicode
                 if self.is_unichar_ptr_or_array():
@@ -131,7 +131,7 @@
                         u = rffi.wcharp2unicode(cdata)
                     else:
                         u = rffi.wcharp2unicoden(cdata, length)
-                    return space.wrap(u)
+                    return space.newunicode(u)
         #
         return W_CType.string(self, cdataobj, maxlen)
 
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py

From pypy.commits at gmail.com  Mon Aug  1 14:46:00 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 01 Aug 2016 11:46:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Remove wrong handle_with_stmt from
 merge
Message-ID: <579f98e8.11051c0a.273b7.5457@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r85969:6f1bd36b0cda
Date: 2016-08-01 20:45 +0200
http://bitbucket.org/pypy/pypy/changeset/6f1bd36b0cda/

Log:	Remove wrong handle_with_stmt from merge

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -415,13 +415,6 @@
         return ast.Try(body, handlers, otherwise, finally_suite,
                        try_node.get_lineno(), try_node.get_column())
 
-    def handle_with_stmt(self, with_node, is_async):
-            if is_async:
-                wi = ast.AsyncWith(test, target, body, with_node.get_lineno(),
-                              with_node.get_column())
-            else:
-                wi = ast.With(test, target, body, with_node.get_lineno(),
-                              with_node.get_column())
     def handle_with_item(self, item_node):
         test = self.handle_expr(item_node.get_child(0))
         if item_node.num_children() == 3:

From pypy.commits at gmail.com  Mon Aug  1 15:23:30 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 12:23:30 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: use brackets to lookup dict
Message-ID: <579fa1b2.d8011c0a.20ece.5623@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r85970:8d8e03f06ed0
Date: 2016-08-01 19:13 +0200
http://bitbucket.org/pypy/pypy/changeset/8d8e03f06ed0/

Log:	use brackets to lookup dict

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -317,7 +317,7 @@
         # register, and we emit a load from the cc into this register.
 
         if resloc is ebp:
-            self.guard_success_cc = condition
+            self.assembler.guard_success_cc = condition
         else:
             assert lhsloc is xmm0
             maskloc = X86_64_XMM_SCRATCH_REG
@@ -334,13 +334,13 @@
             self.mc.CMPPS_xxi(lhsloc.value, rhsloc.value, 1 << 2)
         else:
             self.mc.CMPPD_xxi(lhsloc.value, rhsloc.value, 1 << 2)
-        self.flush_vec_cc(rx86.Conditions("NE"), lhsloc, resloc, sizeloc.value)
+        self.flush_vec_cc(rx86.Conditions["NE"], lhsloc, resloc, sizeloc.value)
 
     def genop_vec_int_eq(self, op, arglocs, resloc):
         lhsloc, rhsloc, sizeloc = arglocs
         size = sizeloc.value
         self.mc.PCMPEQ(lhsloc, rhsloc, size)
-        self.flush_vec_cc(rx86.Conditions("E"), lhsloc, resloc, sizeloc.value)
+        self.flush_vec_cc(rx86.Conditions["E"], lhsloc, resloc, sizeloc.value)
 
     def genop_vec_int_ne(self, op, arglocs, resloc):
         lhsloc, rhsloc, sizeloc = arglocs
@@ -354,7 +354,7 @@
         # 11 11 11 11
         # ----------- pxor
         # 00 11 00 00
-        self.flush_vec_cc(rx86.Conditions("NE"), lhsloc, resloc, sizeloc.value)
+        self.flush_vec_cc(rx86.Conditions["NE"], lhsloc, resloc, sizeloc.value)
 
     def genop_vec_int_signext(self, op, arglocs, resloc):
         srcloc, sizeloc, tosizeloc = arglocs

From pypy.commits at gmail.com  Mon Aug  1 15:23:32 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 12:23:32 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: added three more tests to
 ensure enforce_var_in_vector_reg works (found one issue)
Message-ID: <579fa1b4.56421c0a.23487.5dff@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r85971:a88c5ddefcb0
Date: 2016-08-01 21:22 +0200
http://bitbucket.org/pypy/pypy/changeset/a88c5ddefcb0/

Log:	added three more tests to ensure enforce_var_in_vector_reg works
	(found one issue)

diff --git a/rpython/jit/backend/x86/test/test_x86vector.py b/rpython/jit/backend/x86/test/test_x86vector.py
--- a/rpython/jit/backend/x86/test/test_x86vector.py
+++ b/rpython/jit/backend/x86/test/test_x86vector.py
@@ -1,10 +1,14 @@
 import py
 from rpython.jit.backend.x86.regloc import *
+from rpython.jit.backend.x86.regalloc import (RegAlloc,
+        X86FrameManager, X86XMMRegisterManager, X86RegisterManager)
+from rpython.jit.backend.x86.vector_ext import TempVector
 from rpython.jit.backend.x86.test import test_basic
 from rpython.jit.backend.x86.test.test_assembler import \
         (TestRegallocPushPop as BaseTestAssembler)
 from rpython.jit.metainterp.test import test_vector
 from rpython.rtyper.lltypesystem import lltype
+from rpython.jit.backend.detect_cpu import getcpuclass
 
 class TestBasic(test_basic.Jit386Mixin, test_vector.VectorizeTests):
     # for the individual tests see
@@ -26,6 +30,30 @@
 
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
+ at py.test.fixture
+def regalloc(request):
+    from rpython.jit.backend.x86.regalloc import X86FrameManager
+    from rpython.jit.backend.x86.regalloc import X86XMMRegisterManager
+    class FakeToken:
+        class compiled_loop_token:
+            asmmemmgr_blocks = None
+    cpu = getcpuclass()(None, None)
+    cpu.setup()
+    if cpu.HAS_CODEMAP:
+        cpu.codemap.setup()
+    looptoken = FakeToken()
+    asm = cpu.assembler
+    asm.setup_once()
+    asm.setup(looptoken)
+    regalloc = RegAlloc(asm)
+    regalloc.fm = fm = X86FrameManager(cpu.get_baseofs_of_frame_field())
+    regalloc.rm = X86RegisterManager({}, frame_manager = fm, assembler = asm)
+    regalloc.xrm = X86XMMRegisterManager({}, frame_manager = fm, assembler = asm)
+    request.cls.asm = asm
+    request.cls.regalloc = regalloc
+
+
+
 class TestAssembler(BaseTestAssembler):
 
     def imm_4_int32(self, a, b, c, d):
@@ -88,3 +116,48 @@
         res = self.do_test(callback) & 0xffffffff
         assert res == 22
 
+    def test_enforce_var(self, regalloc):
+        arg = TempVector('f')
+        args = []
+        self.regalloc.fm.bindings[arg] = FrameLoc(0, 64, 'f')
+        reg = self.regalloc.enforce_var_in_vector_reg(arg, args, xmm0)
+        assert reg is xmm0
+
+    def test_enforce_var_xmm0_forbidden(self, regalloc):
+        arg = TempVector('f')
+        arg1 = TempVector('f')
+        args = [arg1]
+        xrm = self.regalloc.xrm
+        xrm.reg_bindings[arg1] = xmm0
+        fr = xrm.free_regs
+        xrm.free_regs = [r for r in fr if r is not xmm0]
+        self.regalloc.fm.bindings[arg] = FrameLoc(0, 64, 'f')
+        reg = self.regalloc.enforce_var_in_vector_reg(arg, args, xmm0)
+        assert reg is xmm0
+        assert len(xrm.reg_bindings) == 2
+        assert xrm.reg_bindings[arg] == xmm0
+        assert xrm.reg_bindings[arg1] != xmm0
+
+    def test_enforce_var_spill(self, regalloc):
+        arg = TempVector('f')
+        arg1 = TempVector('f')
+        arg2 = TempVector('f')
+        args = []
+        xrm = self.regalloc.xrm
+        xrm.reg_bindings[arg1] = xmm0
+        xrm.reg_bindings[arg2] = xmm1
+        xrm.longevity[arg1] = (0,1)
+        xrm.longevity[arg2] = (0,2)
+        xrm.longevity[arg] = (0,3)
+        fr = xrm.free_regs
+        xrm.free_regs = []
+        self.regalloc.fm.bindings[arg] = FrameLoc(0, 64, 'f')
+        self.regalloc.fm.bindings[arg2] = FrameLoc(0, 72, 'f')
+        reg = self.regalloc.enforce_var_in_vector_reg(arg, args, xmm0)
+        assert reg is xmm0
+        assert len(xrm.reg_bindings) == 2
+        assert xrm.reg_bindings[arg] == xmm0
+        assert xrm.reg_bindings[arg1] == xmm1
+        assert arg2 not in xrm.reg_bindings
+
+
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -10,7 +10,7 @@
     xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
     X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, AddressLoc)
 from rpython.jit.backend.llsupport.vector_ext import VectorExt
-from rpython.jit.backend.llsupport.regalloc import get_scale, TempVar
+from rpython.jit.backend.llsupport.regalloc import get_scale, TempVar, NoVariableToSpill
 from rpython.jit.metainterp.resoperation import (rop, ResOperation,
         VectorOp, VectorGuardOp)
 from rpython.rlib.objectmodel import we_are_translated, always_inline
@@ -317,7 +317,7 @@
         # register, and we emit a load from the cc into this register.
 
         if resloc is ebp:
-            self.assembler.guard_success_cc = condition
+            self.guard_success_cc = rev_cond
         else:
             assert lhsloc is xmm0
             maskloc = X86_64_XMM_SCRATCH_REG
@@ -654,7 +654,11 @@
             # do we have a free register?
             if len(xrm.free_regs) == 0:
                 # spill a non forbidden variable
-                self._spill_var(candidate_to_spill, forbidden_vars, None)
+                if not candidate_to_spill:
+                    raise NoVariableToSpill
+                reg = xrm.reg_bindings[candidate_to_spill]
+                xrm._spill_var(candidate_to_spill, forbidden_vars, None)
+                xrm.free_regs.append(reg)
             loc = xrm.free_regs.pop()
             self.assembler.mov(selected_reg, loc)
             reg = xrm.reg_bindings.get(arg, None)

From pypy.commits at gmail.com  Mon Aug  1 15:31:56 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 01 Aug 2016 12:31:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: whoops?
Message-ID: <579fa3ac.54bc1c0a.5a260.5f7f@mx.google.com>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r85972:54055c320d57
Date: 2016-08-01 22:28 +0300
http://bitbucket.org/pypy/pypy/changeset/54055c320d57/

Log:	whoops?

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -46,7 +46,7 @@
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
-    from pypy.tool.error import offset2lineno
+    from rpython.tool.error import offset2lineno
     bcindex = ord(bytecode.co_code[next_instr])
     opname = ""
     if 0 <= bcindex < len(opcode_method_names):

From pypy.commits at gmail.com  Mon Aug  1 16:22:32 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 01 Aug 2016 13:22:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Add note for dirty fix
Message-ID: <579faf88.68adc20a.8208b.0427@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r85973:55d7fbdc81b4
Date: 2016-08-01 22:21 +0200
http://bitbucket.org/pypy/pypy/changeset/55d7fbdc81b4/

Log:	Add note for dirty fix

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1443,7 +1443,7 @@
     def GET_AWAITABLE(self, oparg, next_instr):
         from pypy.objspace.std.noneobject import W_NoneObject
         if isinstance(self.peekvalue(), W_NoneObject):
-            #switch NoneObject with iterable on stack
+            #switch NoneObject with iterable on stack (kind of a dirty fix)
             w_firstnone = self.popvalue()
             w_i = self.popvalue()
             self.pushvalue(w_firstnone)

From pypy.commits at gmail.com  Mon Aug  1 17:26:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 01 Aug 2016 14:26:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: Translation fix
Message-ID: <579fbea1.031dc20a.f2c35.2997@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85974:d046815fc8d5
Date: 2016-08-01 22:35 +0100
http://bitbucket.org/pypy/pypy/changeset/d046815fc8d5/

Log:	Translation fix

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -54,7 +54,7 @@
     name = bytecode.co_name
     if not name:
         name = ""
-    line = offset2lineno(bytecode, next_instr)
+    line = offset2lineno(bytecode, intmask(next_instr))
     return (bytecode.co_filename, line,
             name, intmask(next_instr), opname)
 

From pypy.commits at gmail.com  Mon Aug  1 21:00:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 01 Aug 2016 18:00:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix test_intlike() and more_init case in
 import_extension()
Message-ID: <579ff0bd.274fc20a.39fdf.53be@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r85975:1b2752361da5
Date: 2016-08-02 01:59 +0100
http://bitbucket.org/pypy/pypy/changeset/1b2752361da5/

Log:	fix test_intlike() and more_init case in import_extension()

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -417,8 +417,7 @@
             init = """PyObject *mod = PyModule_Create(&moduledef);"""
             if more_init:
                 init += more_init
-            else:
-                init += "\nreturn mod;"
+            init += "\nreturn mod;"
             return import_module(space, name=modname, init=init, body=body,
                                  w_include_dirs=w_include_dirs,
                                  PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -779,7 +779,7 @@
             """, more_init="""
             IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT;
             IntLike_Type.tp_as_number = &intlike_as_number;
-            intlike_as_number.nb_bool = intlike_nb_nonzero;
+            intlike_as_number.nb_bool = intlike_nb_bool;
             intlike_as_number.nb_int = intlike_nb_int;
             PyType_Ready(&IntLike_Type);
             """)

From pypy.commits at gmail.com  Mon Aug  1 21:18:43 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 01 Aug 2016 18:18:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix bad merge in test_number.py
Message-ID: <579ff4f3.031dc20a.af26a.0223@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r85976:12a7377be119
Date: 2016-08-02 02:18 +0100
http://bitbucket.org/pypy/pypy/changeset/12a7377be119/

Log:	Fix bad merge in test_number.py

diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py
--- a/pypy/module/cpyext/test/test_number.py
+++ b/pypy/module/cpyext/test/test_number.py
@@ -1,5 +1,6 @@
 from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 class TestIterator(BaseApiTest):
     def test_check(self, space, api):
@@ -63,7 +64,9 @@
         assert 9 == space.unwrap(
             api.PyNumber_InPlacePower(space.wrap(3), space.wrap(2), space.w_None))
 
-    def test_PyNumber_Check(self):        
+
+class AppTestCNumber(AppTestCpythonExtensionBase):
+    def test_PyNumber_Check(self):
         mod = self.import_extension('foo', [
             ("test_PyNumber_Check", "METH_VARARGS",
              '''

From pypy.commits at gmail.com  Mon Aug  1 21:54:16 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 01 Aug 2016 18:54:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Port test_import_module.c to 3
Message-ID: <579ffd48.c2f3c20a.a80d5.0919@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r85977:7b64c8a8597e
Date: 2016-08-02 02:53 +0100
http://bitbucket.org/pypy/pypy/changeset/7b64c8a8597e/

Log:	Port test_import_module.c to 3

diff --git a/pypy/module/cpyext/test/test_import_module.c b/pypy/module/cpyext/test/test_import_module.c
--- a/pypy/module/cpyext/test/test_import_module.c
+++ b/pypy/module/cpyext/test/test_import_module.c
@@ -1,17 +1,20 @@
 #include "Python.h"
 /* Initialize this module. */
 
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "test_import_module",
+    NULL,
+    -1,
+    NULL, NULL, NULL, NULL, NULL
+};
+
 PyMODINIT_FUNC
-inittest_import_module(void)
+PyInit_test_import_module(void)
 {
-	PyObject *m, *d;
-
-	m = Py_InitModule("test_import_module", NULL);
-	if (m == NULL)
-	    return;
-	d = PyModule_GetDict(m);
-	if (d) {
-        PyDict_SetItemString(d, "TEST", (PyObject *) Py_None);
-    }
-   	/* No need to check the error here, the caller will do that */
+    PyObject* m = PyModule_Create(&moduledef);
+    if (m == NULL)
+	    return NULL;
+    PyModule_AddObject(m, "TEST", (PyObject *) Py_None);
+    return m;
 }

From pypy.commits at gmail.com  Mon Aug  1 23:53:32 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 01 Aug 2016 20:53:32 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-realloc: test, fix for translated realloc
Message-ID: <57a0193c.ca11c30a.76936.22d7@mx.google.com>

Author: mattip <matti.picus at gmail.com>
Branch: cpyext-realloc
Changeset: r85978:12bb1db94194
Date: 2016-08-02 06:37 +0300
http://bitbucket.org/pypy/pypy/changeset/12bb1db94194/

Log:	test, fix for translated realloc

diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -21,6 +21,8 @@
                          flavor='raw',
                          add_memory_pressure=True)
 
+realloc = rffi.llexternal('realloc', [rffi.VOIDP, rffi.SIZE_T], rffi.VOIDP)
+
 @cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
@@ -28,7 +30,7 @@
                          flavor='raw',
                          add_memory_pressure=True)
     # XXX FIXME
-    return lltype.nullptr(rffi.VOIDP.TO)
+    return realloc(ptr, size)
 
 @cpython_api([rffi.VOIDP], lltype.Void)
 def PyObject_Free(space, ptr):
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -235,7 +235,6 @@
         assert type(x) is int
         assert x == -424344
 
-    @pytest.mark.skipif(True, reason='realloc not fully implemented')
     def test_object_realloc(self):
         module = self.import_extension('foo', [
             ("realloctest", "METH_NOARGS",

From pypy.commits at gmail.com  Tue Aug  2 02:08:25 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 01 Aug 2016 23:08:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: use intmask to not overflow rffi.cast,
 tests pass now again (s390x)
Message-ID: <57a038d9.17a61c0a.d418c.5eda@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r85979:dffc8113a10f
Date: 2016-08-02 08:06 +0200
http://bitbucket.org/pypy/pypy/changeset/dffc8113a10f/

Log:	use intmask to not overflow rffi.cast, tests pass now again (s390x)

diff --git a/rpython/jit/backend/zarch/test/test_assembler.py b/rpython/jit/backend/zarch/test/test_assembler.py
--- a/rpython/jit/backend/zarch/test/test_assembler.py
+++ b/rpython/jit/backend/zarch/test/test_assembler.py
@@ -21,6 +21,7 @@
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.longlong2float import (float2longlong,
         DOUBLE_ARRAY_PTR, singlefloat2uint_emulator)
+from rpython.rlib.rarithmetic import r_uint, intmask
 import ctypes
 
 CPU = getcpuclass()
@@ -168,7 +169,7 @@
     def test_load_byte_zero_extend(self):
         adr = self.a.datablockwrapper.malloc_aligned(16, 16)
         data = rffi.cast(rffi.CArrayPtr(rffi.ULONG), adr)
-        data[0] = rffi.cast(rffi.ULONG,0xffffFFFFffffFF02)
+        data[0] = rffi.cast(rffi.ULONG, intmask(0xffffFFFFffffFF02))
         self.a.mc.load_imm(r.r3, adr+7)
         self.a.mc.LLGC(r.r2, loc.addr(0,r.r3))
         self.a.mc.BCR(con.ANY, r.r14)
@@ -177,7 +178,7 @@
     def test_load_byte_and_imm(self):
         adr = self.a.datablockwrapper.malloc_aligned(16, 16)
         data = rffi.cast(rffi.CArrayPtr(rffi.ULONG), adr)
-        data[0] = rffi.cast(rffi.ULONG,0xffffFFFFffff0001)
+        data[0] = rffi.cast(rffi.ULONG, intmask(0xffffFFFFffff0001))
         self.a.mc.load_imm(r.r3, adr)
         self.a.mc.LG(r.r2, loc.addr(0,r.r3))
         self.a.mc.LLGC(r.r2, loc.addr(7,r.r3))

From pypy.commits at gmail.com  Tue Aug  2 04:41:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 01:41:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: update comment
Message-ID: <57a05cb5.eeb8c20a.6ebe3.7017@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85980:537ba35b03ea
Date: 2016-08-01 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/537ba35b03ea/

Log:	update comment

diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-# hypothesis is used for test generation on untranslated jit tests
+# hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2

From pypy.commits at gmail.com  Tue Aug  2 04:41:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 01:41:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2360: Test and probable fix for
 _rawffi.alt on Windows
Message-ID: <57a05cb7.469d1c0a.c5985.96d4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85981:e87949c507ce
Date: 2016-08-02 10:43 +0200
http://bitbucket.org/pypy/pypy/changeset/e87949c507ce/

Log:	Issue #2360: Test and probable fix for _rawffi.alt on Windows

diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -20,7 +20,8 @@
     def _getfunc(space, CDLL, w_name, w_argtypes, w_restype):
         argtypes_w, argtypes, w_restype, restype = unpack_argtypes(
             space, w_argtypes, w_restype)
-        if space.isinstance_w(w_name, space.w_str):
+        if (space.isinstance_w(w_name, space.w_str) or
+                space.isinstance_w(w_name, space.w_unicode)):
             name = space.str_w(w_name)
             try:
                 func = CDLL.cdll.getpointer(name, argtypes, restype,
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -133,6 +133,12 @@
         # You cannot assing character format codes as restype any longer
         raises(TypeError, setattr, f, "restype", "i")
 
+    def test_unicode_function_name(self):
+        f = dll[u'_testfunc_i_bhilfd']
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_int
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        assert result == 21
 
     def test_truncate_python_longs(self):
         f = dll._testfunc_i_bhilfd

From pypy.commits at gmail.com  Tue Aug  2 04:41:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 01:41:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <57a05cb9.09afc20a.998b.7268@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85982:518912cfcdd9
Date: 2016-08-02 10:43 +0200
http://bitbucket.org/pypy/pypy/changeset/518912cfcdd9/

Log:	merge heads

diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-# hypothesis is used for test generation on untranslated jit tests
+# hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2

From pypy.commits at gmail.com  Tue Aug  2 05:23:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 02:23:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2348: FreeBSD needs this value
 of _POSIX_C_SOURCE to expose
Message-ID: <57a06675.c15e1c0a.4787.4961@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85983:3a63f18f76cb
Date: 2016-08-02 11:24 +0200
http://bitbucket.org/pypy/pypy/changeset/3a63f18f76cb/

Log:	Issue #2348: FreeBSD needs this value of _POSIX_C_SOURCE to expose
	functions like unlinkat(), used on py3k.

diff --git a/rpython/translator/c/src/precommondefs.h b/rpython/translator/c/src/precommondefs.h
--- a/rpython/translator/c/src/precommondefs.h
+++ b/rpython/translator/c/src/precommondefs.h
@@ -18,9 +18,9 @@
 #define _LARGEFILE_SOURCE 1
 /* Define on NetBSD to activate all library features */
 #define _NETBSD_SOURCE 1
-/* Define to activate features from IEEE Stds 1003.1-2001 */
+/* Define to activate features from IEEE Stds 1003.1-2008 */
 #ifndef _POSIX_C_SOURCE
-#  define _POSIX_C_SOURCE 200112L
+#  define _POSIX_C_SOURCE 200809L
 #endif
 /* Define on FreeBSD to activate all library features */
 #define __BSD_VISIBLE 1

From pypy.commits at gmail.com  Tue Aug  2 07:28:19 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 02 Aug 2016 04:28:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Fix import error
 (PyPyClassCollector does not seem to exist anymore,
 pypydir was imported from the wrong file in apptest)
Message-ID: <57a083d3.6814c30a.71dcd.b435@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r85984:cf27d3b81de8
Date: 2016-08-02 13:27 +0200
http://bitbucket.org/pypy/pypy/changeset/cf27d3b81de8/

Log:	Fix import error (PyPyClassCollector does not seem to exist anymore,
	pypydir was imported from the wrong file in apptest)

diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -16,7 +16,7 @@
 from pypy.tool.pytest import appsupport
 from pypy.tool.pytest.objspace import gettestobjspace
 from rpython.tool.udir import udir
-from pypy.conftest import PyPyClassCollector, pypydir
+from pypy import pypydir
 from inspect import getmro
 
 pypyroot = os.path.dirname(pypydir)

From pypy.commits at gmail.com  Tue Aug  2 09:07:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 06:07:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: Manual copy from the
 null_byte_after_str branch: fix a memory leak in cpyext
Message-ID: <57a09b2d.47cbc20a.13c7.d6dc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85985:dafc333097c5
Date: 2016-08-02 15:09 +0200
http://bitbucket.org/pypy/pypy/changeset/dafc333097c5/

Log:	Manual copy from the null_byte_after_str branch: fix a memory leak
	in cpyext

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -96,7 +96,8 @@
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
             py_str.c_ob_size, len(s))
-    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    with rffi.scoped_nonmovingbuffer(s) as s_ptr:
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
     py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL

From pypy.commits at gmail.com  Tue Aug  2 09:26:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 06:26:05 -0700 (PDT)
Subject: [pypy-commit] pypy null_byte_after_str: Revert a few places where
 it doesn't really help. This is also an
Message-ID: <57a09f6d.c2a5c20a.adc42.ddb5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: null_byte_after_str
Changeset: r85986:f4cf210c8a79
Date: 2016-08-02 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/f4cf210c8a79/

Log:	Revert a few places where it doesn't really help. This is also an
	attempt to avoid obscure bugs in code that is not tested a lot.

diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py
--- a/pypy/module/_locale/interp_locale.py
+++ b/pypy/module/_locale/interp_locale.py
@@ -126,9 +126,13 @@
         space.isinstance_w(w_s2, space.w_str)):
 
         s1, s2 = space.str_w(w_s1), space.str_w(w_s2)
-        with rffi.scoped_view_charp(s1) as s1_c:
-            with rffi.scoped_view_charp(s2) as s2_c:
-                return space.wrap(_strcoll(s1_c, s2_c))
+        s1_c = rffi.str2charp(s1)
+        s2_c = rffi.str2charp(s2)
+        try:
+            return space.wrap(_strcoll(s1_c, s2_c))
+        finally:
+            rffi.free_charp(s1_c)
+            rffi.free_charp(s2_c)
 
     s1, s2 = space.unicode_w(w_s1), space.unicode_w(w_s2)
 
@@ -151,15 +155,21 @@
     n1 = len(s) + 1
 
     buf = lltype.malloc(rffi.CCHARP.TO, n1, flavor="raw", zero=True)
-    with rffi.scoped_view_charp(s) as s_c:
+    s_c = rffi.str2charp(s)
+    try:
         n2 = _strxfrm(buf, s_c, n1) + 1
+    finally:
+        rffi.free_charp(s_c)
     if n2 > n1:
         # more space needed
         lltype.free(buf, flavor="raw")
         buf = lltype.malloc(rffi.CCHARP.TO, intmask(n2),
                             flavor="raw", zero=True)
-        with rffi.scoped_view_charp(s) as s_c:
+        s_c = rffi.str2charp(s)
+        try:
             _strxfrm(buf, s_c, n2)
+        finally:
+            rffi.free_charp(s_c)
 
     val = rffi.charp2str(buf)
     lltype.free(buf, flavor="raw")
@@ -188,8 +198,11 @@
     def gettext(space, msg):
         """gettext(msg) -> string
         Return translation of msg."""
-        with rffi.scoped_view_charp(msg) as msg_c:
+        msg_c = rffi.str2charp(msg)
+        try:
             return space.wrap(rffi.charp2str(_gettext(msg_c)))
+        finally:
+            rffi.free_charp(msg_c)
 
     _dgettext = rlocale.external('dgettext', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP)
 
@@ -199,21 +212,28 @@
         Return translation of msg in domain."""
         if space.is_w(w_domain, space.w_None):
             domain = None
-            with rffi.scoped_view_charp(msg) as msg_c:
+            msg_c = rffi.str2charp(msg)
+            try:
                 result = _dgettext(domain, msg_c)
                 # note that 'result' may be the same pointer as 'msg_c',
                 # so it must be converted to an RPython string *before*
                 # we free msg_c.
                 result = rffi.charp2str(result)
+            finally:
+                rffi.free_charp(msg_c)
         else:
             domain = space.str_w(w_domain)
-            with rffi.scoped_view_charp(domain) as domain_c:
-                with rffi.scoped_view_charp(msg) as msg_c:
-                    result = _dgettext(domain_c, msg_c)
-                    # note that 'result' may be the same pointer as 'msg_c',
-                    # so it must be converted to an RPython string *before*
-                    # we free msg_c.
-                    result = rffi.charp2str(result)
+            domain_c = rffi.str2charp(domain)
+            msg_c = rffi.str2charp(msg)
+            try:
+                result = _dgettext(domain_c, msg_c)
+                # note that 'result' may be the same pointer as 'msg_c',
+                # so it must be converted to an RPython string *before*
+                # we free msg_c.
+                result = rffi.charp2str(result)
+            finally:
+                rffi.free_charp(domain_c)
+                rffi.free_charp(msg_c)
 
         return space.wrap(result)
 
@@ -227,22 +247,29 @@
 
         if space.is_w(w_domain, space.w_None):
             domain = None
-            with rffi.scoped_view_charp(msg) as msg_c:
+            msg_c = rffi.str2charp(msg)
+            try:
                 result = _dcgettext(domain, msg_c, rffi.cast(rffi.INT, category))
                 # note that 'result' may be the same pointer as 'msg_c',
                 # so it must be converted to an RPython string *before*
                 # we free msg_c.
                 result = rffi.charp2str(result)
+            finally:
+                rffi.free_charp(msg_c)
         else:
             domain = space.str_w(w_domain)
-            with rffi.scoped_view_charp(domain) as domain_c:
-                with rffi.scoped_view_charp(msg) as msg_c:
-                    result = _dcgettext(domain_c, msg_c,
-                                        rffi.cast(rffi.INT, category))
-                    # note that 'result' may be the same pointer as 'msg_c',
-                    # so it must be converted to an RPython string *before*
-                    # we free msg_c.
-                    result = rffi.charp2str(result)
+            domain_c = rffi.str2charp(domain)
+            msg_c = rffi.str2charp(msg)
+            try:
+                result = _dcgettext(domain_c, msg_c,
+                                    rffi.cast(rffi.INT, category))
+                # note that 'result' may be the same pointer as 'msg_c',
+                # so it must be converted to an RPython string *before*
+                # we free msg_c.
+                result = rffi.charp2str(result)
+            finally:
+                rffi.free_charp(domain_c)
+                rffi.free_charp(msg_c)
 
         return space.wrap(result)
 
@@ -259,12 +286,15 @@
             result = rffi.charp2str(result)
         else:
             domain = space.str_w(w_domain)
-            with rffi.scoped_view_charp(domain) as domain_c:
+            domain_c = rffi.str2charp(domain)
+            try:
                 result = _textdomain(domain_c)
                 # note that 'result' may be the same pointer as 'domain_c'
                 # (maybe?) so it must be converted to an RPython string
                 # *before* we free domain_c.
                 result = rffi.charp2str(result)
+            finally:
+                rffi.free_charp(domain_c)
 
         return space.wrap(result)
 
@@ -279,13 +309,20 @@
 
         if space.is_w(w_dir, space.w_None):
             dir = None
-            with rffi.scoped_view_charp(domain) as domain_c:
+            domain_c = rffi.str2charp(domain)
+            try:
                 dirname = _bindtextdomain(domain_c, dir)
+            finally:
+                rffi.free_charp(domain_c)
         else:
             dir = space.str_w(w_dir)
-            with rffi.scoped_view_charp(domain) as domain_c:
-                with rffi.scoped_view_charp(dir) as dir_c:
-                    dirname = _bindtextdomain(domain_c, dir_c)
+            domain_c = rffi.str2charp(domain)
+            dir_c = rffi.str2charp(dir)
+            try:
+                dirname = _bindtextdomain(domain_c, dir_c)
+            finally:
+                rffi.free_charp(domain_c)
+                rffi.free_charp(dir_c)
 
         if not dirname:
             errno = rposix.get_saved_errno()
@@ -303,13 +340,20 @@
 
             if space.is_w(w_codeset, space.w_None):
                 codeset = None
-                with rffi.scoped_view_charp(domain) as domain_c:
+                domain_c = rffi.str2charp(domain)
+                try:
                     result = _bind_textdomain_codeset(domain_c, codeset)
+                finally:
+                    rffi.free_charp(domain_c)
             else:
                 codeset = space.str_w(w_codeset)
-                with rffi.scoped_view_charp(domain) as domain_c:
-                    with rffi.scoped_view_charp(codeset) as codeset_c:
-                        result = _bind_textdomain_codeset(domain_c, codeset_c)
+                domain_c = rffi.str2charp(domain)
+                codeset_c = rffi.str2charp(codeset)
+                try:
+                    result = _bind_textdomain_codeset(domain_c, codeset_c)
+                finally:
+                    rffi.free_charp(domain_c)
+                    rffi.free_charp(codeset_c)
 
             if not result:
                 return space.w_None
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -52,13 +52,13 @@
         # 1) we automatically get the '\0' sentinel at the end of the string,
         #    which means that we never have to check for the "end of string"
         # 2) we can pass the buffer directly to strtod
-        self.ll_chars, self.buf_flag = rffi.get_nonmovingbuffer_final_null(s)
+        self.ll_chars = rffi.str2charp(s)
         self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         self.pos = 0
         self.last_type = TYPE_UNKNOWN
 
     def close(self):
-        rffi.free_nonmovingbuffer(self.s, self.ll_chars, self.buf_flag)
+        rffi.free_charp(self.ll_chars)
         lltype.free(self.end_ptr, flavor='raw')
 
     def getslice(self, start, end):
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -218,7 +218,7 @@
         subkey = None
     else:
         subkey = space.str_w(w_subkey)
-    with rffi.scoped_view_charp(value) as dataptr:
+    with rffi.scoped_str2charp(value) as dataptr:
         ret = rwinreg.RegSetValue(hkey, subkey, rwinreg.REG_SZ, dataptr, len(value))
         if ret != 0:
             raiseWindowsError(space, ret, 'RegSetValue')
diff --git a/rpython/rlib/_os_support.py b/rpython/rlib/_os_support.py
--- a/rpython/rlib/_os_support.py
+++ b/rpython/rlib/_os_support.py
@@ -20,7 +20,6 @@
     charp2str = staticmethod(rffi.charp2str)
     charpsize2str = staticmethod(rffi.charpsize2str)
     scoped_str2charp = staticmethod(rffi.scoped_str2charp)
-    scoped_view_charp = staticmethod(rffi.scoped_view_charp)
     str2charp = staticmethod(rffi.str2charp)
     free_charp = staticmethod(rffi.free_charp)
     scoped_alloc_buffer = staticmethod(rffi.scoped_alloc_buffer)
@@ -56,8 +55,6 @@
     charpsize2str = staticmethod(rffi.wcharpsize2unicode)
     str2charp = staticmethod(rffi.unicode2wcharp)
     scoped_str2charp = staticmethod(rffi.scoped_unicode2wcharp)
-    scoped_view_charp = staticmethod(rffi.scoped_unicode2wcharp)
-    # ^^^ XXX there is no unicode variant of rffi.scoped_view_charp
     free_charp = staticmethod(rffi.free_wcharp)
     scoped_alloc_buffer = staticmethod(rffi.scoped_alloc_unicodebuffer)
 
diff --git a/rpython/rlib/rposix_environ.py b/rpython/rlib/rposix_environ.py
--- a/rpython/rlib/rposix_environ.py
+++ b/rpython/rlib/rposix_environ.py
@@ -163,7 +163,7 @@
         return result
 
     def getenv_llimpl(name):
-        with traits.scoped_view_charp(name) as l_name:
+        with traits.scoped_str2charp(name) as l_name:
             l_result = getenv(l_name)
             return traits.charp2str(l_result) if l_result else None
 
@@ -206,7 +206,7 @@
                                   save_err=rffi.RFFI_SAVE_ERRNO)
 
     def unsetenv_llimpl(name):
-        with rffi.scoped_view_charp(name) as l_name:
+        with rffi.scoped_str2charp(name) as l_name:
             error = rffi.cast(lltype.Signed, os_unsetenv(l_name))
         if error:
             from rpython.rlib import rposix
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -963,7 +963,7 @@
         self.settimeout(timeout)
 
     def setsockopt(self, level, option, value):
-        with rffi.scoped_view_charp(value) as buf:
+        with rffi.scoped_str2charp(value) as buf:
             res = _c.socketsetsockopt(self.fd, level, option,
                                       rffi.cast(rffi.VOIDP, buf),
                                       len(value))

From pypy.commits at gmail.com  Tue Aug  2 09:26:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 06:26:07 -0700 (PDT)
Subject: [pypy-commit] pypy null_byte_after_str: Cancel a fragile dependency
 with details of space.str_w(), and instead
Message-ID: <57a09f6f.87941c0a.985f8.01eb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: null_byte_after_str
Changeset: r85987:2977ce236abb
Date: 2016-08-02 15:27 +0200
http://bitbucket.org/pypy/pypy/changeset/2977ce236abb/

Log:	Cancel a fragile dependency with details of space.str_w(), and
	instead do it all inside W_CTypeFunc._call

diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -157,11 +157,13 @@
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
         try:
+            keepalives = [None] * len(args_w)    # None or strings
             for i in range(len(args_w)):
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                 w_obj = args_w[i]
                 argtype = self.fargs[i]
-                if argtype.convert_argument_from_object(data, w_obj):
+                if argtype.convert_argument_from_object(data, w_obj,
+                                                        keepalives, i):
                     # argtype is a pointer type, and w_obj a list/tuple/str
                     mustfree_max_plus_1 = i + 1
 
@@ -181,7 +183,8 @@
                     if flag == 1:
                         lltype.free(raw_cdata, flavor='raw')
                     elif flag >= 4:
-                        value = args_w[i].str_w(space)
+                        value = keepalives[i]
+                        assert value is not None
                         rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
             lltype.free(buffer, flavor='raw')
             keepalive_until_here(args_w)
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -83,7 +83,7 @@
         raise oefmt(space.w_TypeError, "cannot initialize cdata '%s'",
                     self.name)
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         self.convert_from_object(cdata, w_ob)
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -261,13 +261,12 @@
         else:
             return lltype.nullptr(rffi.CCHARP.TO)
 
-    def _prepare_pointer_call_argument(self, w_init, cdata):
+    def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
         space = self.space
         if self.accept_str and space.isinstance_w(w_init, space.w_str):
             # special case to optimize strings passed to a "char *" argument
-            # WARNING: this relies on the fact that w_init.str_w() returns
-            # always the same object for the same w_init!
             value = w_init.str_w(space)
+            keepalives[i] = value
             buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
             rffi.cast(rffi.CCHARPP, cdata)[0] = buf
             return ord(buf_flag)    # 4, 5 or 6
@@ -307,10 +306,11 @@
         rffi.cast(rffi.CCHARPP, cdata)[0] = result
         return 1
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
         result = (not isinstance(w_ob, cdataobj.W_CData) and
-                  self._prepare_pointer_call_argument(w_ob, cdata))
+                  self._prepare_pointer_call_argument(w_ob, cdata,
+                                                      keepalives, i))
         if result == 0:
             self.convert_from_object(cdata, w_ob)
         set_mustfree_flag(cdata, result)
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -451,9 +451,6 @@
         return self._value
 
     def str_w(self, space):
-        # WARNING: _cffi_backend/ctypeptr.py depends on the fact that
-        # w_obj.str_w() called twice on the same object returns the
-        # exact same string object!
         return self._value
 
     def buffer_w(self, space, flags):

From pypy.commits at gmail.com  Tue Aug  2 09:48:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 Aug 2016 06:48:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix test_obj.py
Message-ID: <57a0a4c4.c3f0c20a.4a412.e8d4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r85988:381e9e66fcc3
Date: 2016-08-02 14:48 +0100
http://bitbucket.org/pypy/pypy/changeset/381e9e66fcc3/

Log:	fix test_obj.py

diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -185,12 +185,12 @@
             skip("cannot run this test as apptest")
         for u in [u"", u"a", u"aa"]:
             assert id(self.unwrap_wrap_unicode(u)) == id(u)
-            s = str(u)
-            assert id(self.unwrap_wrap_str(s)) == id(s)
+            s = u.encode()
+            assert id(self.unwrap_wrap_bytes(s)) == id(s)
         #
-        assert id('') == (256 << 4) | 11     # always
+        assert id(b'') == (256 << 4) | 11     # always
         assert id(u'') == (257 << 4) | 11
-        assert id('a') == (ord('a') << 4) | 11
+        assert id(b'a') == (ord('a') << 4) | 11
         assert id(u'\u1234') == ((~0x1234) << 4) | 11
 
     def test_id_of_tuples(self):
@@ -243,13 +243,13 @@
         l = []
         def add(s, u):
             l.append(s)
-            l.append(self.unwrap_wrap_str(s))
+            l.append(self.unwrap_wrap_bytes(s))
             l.append(s[:1] + s[1:])
             l.append(u)
             l.append(self.unwrap_wrap_unicode(u))
             l.append(u[:1] + u[1:])
         for i in range(3, 18):
-            add(str(i), unicode(i))
+            add(str(i).encode(), str(i))
         add(b"s", u"s")
         add(b"", u"")
 

From pypy.commits at gmail.com  Tue Aug  2 09:55:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 Aug 2016 06:55:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: The empty string hash is now -2
Message-ID: <57a0a66a.497bc20a.273c8.eece@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r85989:a215c6257e88
Date: 2016-08-02 14:55 +0100
http://bitbucket.org/pypy/pypy/changeset/a215c6257e88/

Log:	The empty string hash is now -2

diff --git a/lib-python/3/test/test_hash.py b/lib-python/3/test/test_hash.py
--- a/lib-python/3/test/test_hash.py
+++ b/lib-python/3/test/test_hash.py
@@ -164,7 +164,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 

From pypy.commits at gmail.com  Tue Aug  2 12:11:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 09:11:35 -0700 (PDT)
Subject: [pypy-commit] pypy null_byte_after_str: close branch, ready to merge
Message-ID: <57a0c637.cb7f1c0a.62e3c.d401@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: null_byte_after_str
Changeset: r85990:69db7529f95b
Date: 2016-08-02 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/69db7529f95b/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Tue Aug  2 12:11:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 09:11:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge null_byte_after_str
Message-ID: <57a0c63a.cb7f1c0a.62e3c.d406@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85991:a84c4b359dcc
Date: 2016-08-02 18:13 +0200
http://bitbucket.org/pypy/pypy/changeset/a84c4b359dcc/

Log:	hg merge null_byte_after_str

	Allocate all RPython strings with one extra byte, normally unused.
	It is used to hold a final zero in case we need some 'char *'
	representation of the string, together with checks like 'not
	can_move()' or object pinning. Main new thing that this allows:
	'ffi.from_buffer(string)'.

diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -157,11 +157,13 @@
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
         try:
+            keepalives = [None] * len(args_w)    # None or strings
             for i in range(len(args_w)):
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                 w_obj = args_w[i]
                 argtype = self.fargs[i]
-                if argtype.convert_argument_from_object(data, w_obj):
+                if argtype.convert_argument_from_object(data, w_obj,
+                                                        keepalives, i):
                     # argtype is a pointer type, and w_obj a list/tuple/str
                     mustfree_max_plus_1 = i + 1
 
@@ -177,9 +179,13 @@
                 if isinstance(argtype, W_CTypePointer):
                     data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                     flag = get_mustfree_flag(data)
+                    raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                     if flag == 1:
-                        raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
+                    elif flag >= 4:
+                        value = keepalives[i]
+                        assert value is not None
+                        rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
             lltype.free(buffer, flavor='raw')
             keepalive_until_here(args_w)
         return w_res
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -83,7 +83,7 @@
         raise oefmt(space.w_TypeError, "cannot initialize cdata '%s'",
                     self.name)
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         self.convert_from_object(cdata, w_ob)
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,8 +14,8 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
+    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
     length = -1
 
     def __init__(self, space, size, extra, extra_position, ctitem,
@@ -28,6 +28,9 @@
         #  - for functions, it is the return type
         self.ctitem = ctitem
         self.can_cast_anything = could_cast_anything and ctitem.cast_anything
+        self.accept_str = (self.can_cast_anything or
+                            (ctitem.is_primitive_integer and
+                             ctitem.size == rffi.sizeof(lltype.Char)))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -70,9 +73,7 @@
                 pass
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
-        elif (self.can_cast_anything or
-              (self.ctitem.is_primitive_integer and
-               self.ctitem.size == rffi.sizeof(lltype.Char))):
+        elif self.accept_str:
             if not space.isinstance_w(w_ob, space.w_str):
                 raise self._convert_error("str or list or tuple", w_ob)
             s = space.str_w(w_ob)
@@ -260,8 +261,16 @@
         else:
             return lltype.nullptr(rffi.CCHARP.TO)
 
-    def _prepare_pointer_call_argument(self, w_init, cdata):
+    def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
         space = self.space
+        if self.accept_str and space.isinstance_w(w_init, space.w_str):
+            # special case to optimize strings passed to a "char *" argument
+            value = w_init.str_w(space)
+            keepalives[i] = value
+            buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
+            rffi.cast(rffi.CCHARPP, cdata)[0] = buf
+            return ord(buf_flag)    # 4, 5 or 6
+        #
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
@@ -297,10 +306,11 @@
         rffi.cast(rffi.CCHARPP, cdata)[0] = result
         return 1
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
         result = (not isinstance(w_ob, cdataobj.W_CData) and
-                  self._prepare_pointer_call_argument(w_ob, cdata))
+                  self._prepare_pointer_call_argument(w_ob, cdata,
+                                                      keepalives, i))
         if result == 0:
             self.convert_from_object(cdata, w_ob)
         set_mustfree_flag(cdata, result)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -353,7 +353,7 @@
 'array.array' or numpy arrays."""
         #
         w_ctchara = newtype._new_chara_type(self.space)
-        return func.from_buffer(self.space, w_ctchara, w_python_buffer)
+        return func._from_buffer(self.space, w_ctchara, w_python_buffer)
 
 
     @unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated
+from rpython.rlib import jit
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -132,17 +133,66 @@
         raise oefmt(space.w_TypeError,
                     "needs 'char[]', got '%s'", w_ctype.name)
     #
+    return _from_buffer(space, w_ctype, w_x)
+
+def _from_buffer(space, w_ctype, w_x):
     buf = _fetch_as_read_buffer(space, w_x)
-    try:
-        _cdata = buf.get_raw_address()
-    except ValueError:
-        raise oefmt(space.w_TypeError,
-                    "from_buffer() got a '%T' object, which supports the "
-                    "buffer interface but cannot be rendered as a plain "
-                    "raw address on PyPy", w_x)
+    if space.isinstance_w(w_x, space.w_str):
+        _cdata = get_raw_address_of_string(space, w_x)
+    else:
+        try:
+            _cdata = buf.get_raw_address()
+        except ValueError:
+            raise oefmt(space.w_TypeError,
+                        "from_buffer() got a '%T' object, which supports the "
+                        "buffer interface but cannot be rendered as a plain "
+                        "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
 
+# ____________________________________________________________
+
+class RawBytes(object):
+    def __init__(self, string):
+        self.ptr = rffi.str2charp(string, track_allocation=False)
+    def __del__(self):
+        rffi.free_charp(self.ptr, track_allocation=False)
+
+class RawBytesCache(object):
+    def __init__(self, space):
+        from pypy.interpreter.baseobjspace import W_Root
+        from rpython.rlib import rweakref
+        self.wdict = rweakref.RWeakKeyDictionary(W_Root, RawBytes)
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(space, w_x):
+    """Special case for ffi.from_buffer(string).  Returns a 'char *' that
+    is valid as long as the string object is alive.  Two calls to
+    ffi.from_buffer(same_string) are guaranteed to return the same pointer.
+    """
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rtyper.lltypesystem.rstr import STR
+    from rpython.rtyper.lltypesystem import llmemory
+    from rpython.rlib import rgc
+
+    cache = space.fromcache(RawBytesCache)
+    rawbytes = cache.wdict.get(w_x)
+    if rawbytes is None:
+        data = space.str_w(w_x)
+        if we_are_translated() and not rgc.can_move(data):
+            lldata = llstr(data)
+            data_start = (llmemory.cast_ptr_to_adr(lldata) +
+                          rffi.offsetof(STR, 'chars') +
+                          llmemory.itemoffsetof(STR.chars, 0))
+            data_start = rffi.cast(rffi.CCHARP, data_start)
+            data_start[len(data)] = '\x00'   # write the final extra null
+            return data_start
+        rawbytes = RawBytes(data)
+        cache.wdict.set(w_x, rawbytes)
+    return rawbytes.ptr
+
+# ____________________________________________________________
+
 
 def unsafe_escaping_ptr_for_ptr_or_array(w_cdata):
     if not w_cdata.ctype.is_nonfunc_pointer_or_array:
diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py
--- a/pypy/module/_cffi_backend/parse_c_type.py
+++ b/pypy/module/_cffi_backend/parse_c_type.py
@@ -97,11 +97,8 @@
                                   [rffi.INT], rffi.CCHARP)
 
 def parse_c_type(info, input):
-    p_input = rffi.str2charp(input)
-    try:
+    with rffi.scoped_view_charp(input) as p_input:
         res = ll_parse_c_type(info, p_input)
-    finally:
-        rffi.free_charp(p_input)
     return rffi.cast(lltype.Signed, res)
 
 NULL_CTX = lltype.nullptr(PCTX.TO)
@@ -130,15 +127,13 @@
     return rffi.getintfield(src_ctx, 'c_num_types')
 
 def search_in_globals(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_globals(ctx, c_name,
-                                  rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_globals(ctx, c_name,
+                                      rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
 
 def search_in_struct_unions(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_struct_unions(ctx, c_name,
-                                        rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_struct_unions(ctx, c_name,
+                                            rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3330,13 +3330,18 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharA = new_array_type(BCharP, None)
-    py.test.raises(TypeError, from_buffer, BCharA, b"foo")
+    p1 = from_buffer(BCharA, b"foo")
+    assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # from_buffer(buffer(b"foo")) does not work, because it's not
+        # implemented on pypy; only from_buffer(b"foo") works.
         py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
         py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
     try:
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -401,21 +401,20 @@
             _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
         from rpython.rlib import rwin32
 
-        charp = rffi.str2charp(buf)
-        written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                    flavor='raw')
-        try:
-            result = _WriteFile(
-                self.handle, rffi.ptradd(charp, offset),
-                size, written_ptr, rffi.NULL)
+        with rffi.scoped_view_charp(buf) as charp:
+            written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
+                                        flavor='raw')
+            try:
+                result = _WriteFile(
+                    self.handle, rffi.ptradd(charp, offset),
+                    size, written_ptr, rffi.NULL)
 
-            if (result == 0 and
-                rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
-                raise oefmt(space.w_ValueError,
-                            "Cannot send %d bytes over connection", size)
-        finally:
-            rffi.free_charp(charp)
-            lltype.free(written_ptr, flavor='raw')
+                if (result == 0 and
+                    rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
+                    raise oefmt(space.w_ValueError,
+                                "Cannot send %d bytes over connection", size)
+            finally:
+                lltype.free(written_ptr, flavor='raw')
 
     def do_recv_string(self, space, buflength, maxlength):
         from pypy.module._multiprocessing.interp_win32 import (
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -135,7 +135,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
@@ -147,7 +147,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
@@ -181,7 +181,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
@@ -193,7 +193,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
@@ -228,7 +228,7 @@
 
         Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
         bound on the entropy contained in string."""
-        with rffi.scoped_str2charp(string) as buf:
+        with rffi.scoped_nonmovingbuffer(string) as buf:
             libssl_RAND_add(buf, len(string), entropy)
 
     def RAND_status(space):
diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -537,9 +537,8 @@
     releasegil=ts_helper,
     compilation_info=backend.eci)
 def c_charp2stdstring(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2stdstring(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2stdstring(charp)
     return result
 _c_stdstring2stdstring = rffi.llexternal(
     "cppyy_stdstring2stdstring",
diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -82,9 +82,8 @@
     releasegil=ts_helper,
     compilation_info=eci)
 def c_charp2TString(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2TString(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2TString(charp)
     return result
 _c_TString2TString = rffi.llexternal(
     "cppyy_TString2TString",
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -65,6 +65,7 @@
                 else:    # only other use is sring
                     n = len(obj._string)
                     assert raw_string == rffi.cast(rffi.CCHARP, 0)
+                    # XXX could use rffi.get_nonmovingbuffer_final_null()
                     raw_string = rffi.str2charp(obj._string)
                     data = rffi.cast(rffi.CCHARPP, data)
                     data[0] = raw_string
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -883,6 +883,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                               self.cpu.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -280,7 +280,7 @@
     concrete_type = '\x00'
 
     def __init__(self, basesize, itemsize, lendescr, flag, is_pure=False, concrete_type='\x00'):
-        self.basesize = basesize
+        self.basesize = basesize    # this includes +1 for STR
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
@@ -676,7 +676,7 @@
 
 def unpack_arraydescr(arraydescr):
     assert isinstance(arraydescr, ArrayDescr)
-    ofs = arraydescr.basesize
+    ofs = arraydescr.basesize    # this includes +1 for STR
     size = arraydescr.itemsize
     sign = arraydescr.is_item_signed()
     return size, ofs, sign
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -293,6 +293,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_load_or_indexed(op, op.getarg(0), op.getarg(1),
                                          itemsize, itemsize, basesize, NOT_SIGNED)
         elif opnum == rop.UNICODEGETITEM:
@@ -304,6 +305,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_store_or_indexed(op, op.getarg(0), op.getarg(1), op.getarg(2),
                                          itemsize, itemsize, basesize)
         elif opnum == rop.UNICODESETITEM:
diff --git a/rpython/jit/backend/llsupport/symbolic.py b/rpython/jit/backend/llsupport/symbolic.py
--- a/rpython/jit/backend/llsupport/symbolic.py
+++ b/rpython/jit/backend/llsupport/symbolic.py
@@ -29,7 +29,7 @@
 def get_array_token(T, translate_support_code):
     # T can be an array or a var-sized structure
     if translate_support_code:
-        basesize = llmemory.sizeof(T, 0)
+        basesize = llmemory.sizeof(T, 0)     # this includes +1 for STR
         if isinstance(T, lltype.Struct):
             SUBARRAY = getattr(T, T._arrayfld)
             itemsize = llmemory.sizeof(SUBARRAY.OF)
@@ -57,6 +57,7 @@
             assert carray.length.size == WORD
             ofs_length = before_array_part + carray.length.offset
         basesize = before_array_part + carray.items.offset
+        basesize += T._hints.get('extra_item_after_alloc', 0)  # +1 for STR
         carrayitem = ll2ctypes.get_ctypes_type(T.OF)
         itemsize = ctypes.sizeof(carrayitem)
     return basesize, itemsize, ofs_length
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -435,8 +435,10 @@
 def test_bytearray_descr():
     c0 = GcCache(False)
     descr = get_array_descr(c0, rstr.STR)   # for bytearray
+    # note that we get a basesize that has 1 extra byte for the final null char
+    # (only for STR)
     assert descr.flag == FLAG_UNSIGNED
-    assert descr.basesize == struct.calcsize("PP")         # hash, length
+    assert descr.basesize == struct.calcsize("PP") + 1     # hash, length, extra
     assert descr.lendescr.offset == struct.calcsize("P")   # hash
     assert not descr.is_array_of_pointers()
 
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -647,6 +647,9 @@
         """)
 
     def test_rewrite_assembler_newstr_newunicode(self):
+        # note: strdescr.basesize already contains the extra final character,
+        # so that's why newstr(14) is rounded up to 'basesize+15' and not
+        # 'basesize+16'.
         self.check_rewrite("""
             [i2]
             p0 = newstr(14)
@@ -657,12 +660,12 @@
         """, """
             [i2]
             p0 = call_malloc_nursery(                                \
-                      %(strdescr.basesize + 16 * strdescr.itemsize + \
+                      %(strdescr.basesize + 15 * strdescr.itemsize + \
                         unicodedescr.basesize + 10 * unicodedescr.itemsize)d)
             gc_store(p0, 0,  %(strdescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p0, %(strlendescr.offset)s, 14, %(strlendescr.field_size)s)
             gc_store(p0, 0,  0, %(strhashdescr.field_size)s)
-            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
+            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 15 * strdescr.itemsize)d)
             gc_store(p1, 0,  %(unicodedescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p1, %(unicodelendescr.offset)s, 10, %(unicodelendescr.field_size)s)
             gc_store(p1, 0,  0, %(unicodehashdescr.field_size)s)
@@ -1240,14 +1243,14 @@
         #               'i3 = gc_load_i(p0,i5,%(unicodedescr.itemsize)d)'],
         [True,  (4,),  'i3 = strgetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,1,'
-                       '%(strdescr.basesize)d,1)'],
+                       '%(strdescr.basesize-1)d,1)'],
         #[False, (4,),  'i3 = strgetitem(p0,i1)' '->'
-        #               'i5 = int_add(i1, %(strdescr.basesize)d);'
+        #               'i5 = int_add(i1, %(strdescr.basesize-1)d);'
         #               'i3 = gc_load_i(p0,i5,1)'],
         ## setitem str/unicode
         [True, (4,),  'i3 = strsetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,1,'
-                               '%(strdescr.basesize)d,1)'],
+                               '%(strdescr.basesize-1)d,1)'],
         [True, (2,4), 'i3 = unicodesetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,'
                                  '%(unicodedescr.itemsize)d,'
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -3,7 +3,7 @@
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
 from rpython.rlib.jit import promote, _get_virtualizable_token
-from rpython.rlib import jit_hooks, rposix
+from rpython.rlib import jit_hooks, rposix, rgc
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
 from rpython.jit.backend.detect_cpu import getcpuclass
@@ -11,7 +11,7 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 from rpython.config.config import ConfigError
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.rlib.rjitlog import rjitlog as jl
 
 
@@ -29,6 +29,7 @@
         # - floats neg and abs
         # - cast_int_to_float
         # - llexternal with macro=True
+        # - extra place for the zero after STR instances
 
         class BasicFrame(object):
             _virtualizable_ = ['i']
@@ -56,7 +57,7 @@
             return ("/home.py",0,0)
 
         jitdriver = JitDriver(greens = [],
-                              reds = ['total', 'frame', 'j'],
+                              reds = ['total', 'frame', 'prev_s', 'j'],
                               virtualizables = ['frame'],
                               get_location = get_location)
         def f(i, j):
@@ -68,9 +69,12 @@
             total = 0
             frame = Frame(i)
             j = float(j)
+            prev_s = rstr.mallocstr(16)
             while frame.i > 3:
-                jitdriver.can_enter_jit(frame=frame, total=total, j=j)
-                jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                jitdriver.can_enter_jit(frame=frame, total=total, j=j,
+                                        prev_s=prev_s)
+                jitdriver.jit_merge_point(frame=frame, total=total, j=j,
+                                          prev_s=prev_s)
                 _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
@@ -82,6 +86,11 @@
                 k = myabs1(myabs2(j))
                 if k - abs(j):  raise ValueError
                 if k - abs(-j): raise ValueError
+                s = rstr.mallocstr(16)
+                rgc.ll_write_final_null_char(s)
+                rgc.ll_write_final_null_char(prev_s)
+                if (frame.i & 3) == 0:
+                    prev_s = s
             return chr(total % 253)
         #
         class Virt2(object):
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -994,6 +994,7 @@
             basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                         self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             scale = 0
 
         self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1673,25 +1673,6 @@
         dest_addr = AddressLoc(base_loc, ofs_loc, scale, offset_loc.value)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
-    def genop_discard_strsetitem(self, op, arglocs):
-        base_loc, ofs_loc, val_loc = arglocs
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                              self.cpu.translate_support_code)
-        assert itemsize == 1
-        dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize)
-        self.mc.MOV8(dest_addr, val_loc.lowest8bits())
-
-    def genop_discard_unicodesetitem(self, op, arglocs):
-        base_loc, ofs_loc, val_loc = arglocs
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                              self.cpu.translate_support_code)
-        if itemsize == 4:
-            self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc)
-        elif itemsize == 2:
-            self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc)
-        else:
-            assert 0, itemsize
-
     # genop_discard_setfield_raw = genop_discard_setfield_gc
 
     def genop_math_read_timestamp(self, op, arglocs, resloc):
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1219,6 +1219,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                                   self.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self.assembler.load_effective_addr(ofsloc, ofs_items, scale,
                                            resloc, baseloc)
diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py
--- a/rpython/jit/backend/zarch/opassembler.py
+++ b/rpython/jit/backend/zarch/opassembler.py
@@ -991,6 +991,7 @@
             basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                         self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             scale = 0
 
         # src and src_len are tmp registers
diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -1381,7 +1381,7 @@
             return result
 
         def indirection(arg):
-            return interp(arg)
+            return interp(arg) + 1
 
         def run_interp(n):
             f = hint(Frame(n), access_directly=True)
diff --git a/rpython/memory/gcheader.py b/rpython/memory/gcheader.py
--- a/rpython/memory/gcheader.py
+++ b/rpython/memory/gcheader.py
@@ -11,7 +11,21 @@
     def __init__(self, HDR):
         """NOT_RPYTHON"""
         self.HDR = HDR
-        self.obj2header = weakref.WeakKeyDictionary()
+        #
+        # The following used to be a weakref.WeakKeyDictionary(), but
+        # the problem is that if you have a gcobj which has already a
+        # weakref cached on it and the hash already cached in that
+        # weakref, and later the hash of the gcobj changes (because it
+        # is ll2ctypes-ified), then that gcobj cannot be used as a key
+        # in a WeakKeyDictionary any more: from this point on,
+        # 'ref(gcobj)' and 'ref(gcobj, callback)' return two objects
+        # with different hashes... and so e.g. the sequence of
+        # operations 'obj2header[x]=y; assert x in obj2header' fails.
+        #
+        # Instead, just use a regular dictionary and hope that not too
+        # many objects would be reclaimed in a given GCHeaderBuilder
+        # instance.
+        self.obj2header = {}
         self.size_gc_header = llmemory.GCHeaderOffset(self)
 
     def header_of_object(self, gcptr):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -281,6 +281,10 @@
         return lltype.Signed
 
 malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0)
+_translated_to_c = CDefinedIntSymbolic('1 /*_translated_to_c*/', default=0)
+
+def we_are_translated_to_c():
+    return we_are_translated() and _translated_to_c
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py
--- a/rpython/rlib/rdtoa.py
+++ b/rpython/rlib/rdtoa.py
@@ -56,22 +56,24 @@
         raise MemoryError
     end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
     try:
-        ll_input = rffi.str2charp(input)
+        # note: don't use the class scoped_view_charp here, it
+        # break some tests because this function is used by the GC
+        ll_input, flag = rffi.get_nonmovingbuffer_final_null(input)
         try:
             result = dg_strtod(ll_input, end_ptr)
 
             endpos = (rffi.cast(lltype.Signed, end_ptr[0]) -
                       rffi.cast(lltype.Signed, ll_input))
-
-            if endpos == 0 or endpos < len(input):
-                raise ValueError("invalid input at position %d" % (endpos,))
-
-            return result
         finally:
-            rffi.free_charp(ll_input)
+            rffi.free_nonmovingbuffer(input, ll_input, flag)
     finally:
         lltype.free(end_ptr, flavor='raw')
 
+    if endpos == 0 or endpos < len(input):
+        raise ValueError("invalid input at position %d" % (endpos,))
+
+    return result
+
 lower_special_strings = ['inf', '+inf', '-inf', 'nan']
 upper_special_strings = ['INF', '+INF', '-INF', 'NAN']
 
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1268,3 +1268,26 @@
     ptr = lltype.direct_arrayitems(array)
     # ptr is a Ptr(FixedSizeArray(Char, 1)).  Cast it to a rffi.CCHARP
     return rffi.cast(rffi.CCHARP, ptr)
+
+ at jit.dont_look_inside
+ at no_collect
+ at specialize.ll()
+def ll_write_final_null_char(s):
+    """'s' is a low-level STR; writes a terminating NULL character after
+    the other characters in 's'.  Warning, this only works because of
+    the 'extra_item_after_alloc' hack inside the definition of STR.
+    """
+    from rpython.rtyper.lltypesystem import rffi
+    PSTR = lltype.typeOf(s)
+    assert has_final_null_char(PSTR) == 1
+    n = llmemory.offsetof(PSTR.TO, 'chars')
+    n += llmemory.itemoffsetof(PSTR.TO.chars, 0)
+    n = llmemory.raw_malloc_usage(n)
+    n += len(s.chars)
+    # no GC operation from here!
+    ptr = rffi.cast(rffi.CCHARP, s)
+    ptr[n] = '\x00'
+
+ at specialize.memo()
+def has_final_null_char(PSTR):
+    return PSTR.TO.chars._hints.get('extra_item_after_alloc', 0)
diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py
--- a/rpython/rtyper/lltypesystem/ll2ctypes.py
+++ b/rpython/rtyper/lltypesystem/ll2ctypes.py
@@ -250,7 +250,9 @@
 
         if not A._hints.get('nolength'):
             _fields_ = [('length', lentype),
-                        ('items',  max_n * ctypes_item)]
+                        ('items',
+                           (max_n + A._hints.get('extra_item_after_alloc', 0))
+                           * ctypes_item)]
         else:
             _fields_ = [('items',  max_n * ctypes_item)]
 
@@ -695,6 +697,9 @@
         # we have no clue, so we allow whatever index
         return 0, maxint
 
+    def shrinklength(self, newlength):
+        raise NotImplementedError
+
     def getitem(self, index, uninitialized_ok=False):
         res = self._storage.contents._getitem(index, boundscheck=False)
         if isinstance(self._TYPE.OF, lltype.ContainerType):
diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py
--- a/rpython/rtyper/lltypesystem/llmemory.py
+++ b/rpython/rtyper/lltypesystem/llmemory.py
@@ -304,8 +304,15 @@
         return cast_ptr_to_adr(p)
 
     def raw_memcopy(self, srcadr, dstadr):
-        # should really copy the length field, but we can't
-        pass
+        # copy the length field, if we can
+        srclen = srcadr.ptr._obj.getlength()
+        dstlen = dstadr.ptr._obj.getlength()
+        if dstlen != srclen:
+            assert dstlen > srclen, "can't increase the length"
+            # a decrease in length occurs in the GC tests when copying a STR:
+            # the copy is initially allocated with really one extra char,
+            # the 'extra_item_after_alloc', and must be fixed.
+            dstadr.ptr._obj.shrinklength(srclen)
 
 
 class ArrayLengthOffset(AddressOffset):
@@ -390,11 +397,23 @@
     else:
         raise Exception("don't know how to take the size of a %r"%TYPE)
 
+ at specialize.memo()
+def extra_item_after_alloc(ARRAY):
+    assert isinstance(ARRAY, lltype.Array)
+    return ARRAY._hints.get('extra_item_after_alloc', 0)
+
 @specialize.arg(0)
 def sizeof(TYPE, n=None):
+    """Return the symbolic size of TYPE.
+    For a Struct with no varsized part, it must be called with n=None.
+    For an Array or a Struct with a varsized part, it is the number of items.
+    There is a special case to return 1 more than requested if the array
+    has the hint 'extra_item_after_alloc' set to 1.
+    """
     if n is None:
         return _sizeof_none(TYPE)
     elif isinstance(TYPE, lltype.Array):
+        n += extra_item_after_alloc(TYPE)
         return itemoffsetof(TYPE) + _sizeof_none(TYPE.OF) * n
     else:
         return _sizeof_int(TYPE, n)
@@ -1036,7 +1055,7 @@
                 _reccopy(subsrc, subdst)
             else:
                 # this is a hack XXX de-hack this
-                llvalue = source._obj.getitem(i, uninitialized_ok=True)
+                llvalue = source._obj.getitem(i, uninitialized_ok=2)
                 if not isinstance(llvalue, lltype._uninitialized):
                     dest._obj.setitem(i, llvalue)
     elif isinstance(T, lltype.Struct):
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1926,14 +1926,29 @@
         return 0, stop
 
     def getitem(self, index, uninitialized_ok=False):
-        v = self.items[index]
+        try:
+            v = self.items[index]
+        except IndexError:
+            if (index == len(self.items) and uninitialized_ok == 2 and
+                self._TYPE._hints.get('extra_item_after_alloc')):
+                # special case: reading the extra final char returns
+                # an uninitialized, if 'uninitialized_ok==2'
+                return _uninitialized(self._TYPE.OF)
+            raise
         if isinstance(v, _uninitialized) and not uninitialized_ok:
             raise UninitializedMemoryAccess("%r[%s]"%(self, index))
         return v
 
     def setitem(self, index, value):
         assert typeOf(value) == self._TYPE.OF
-        self.items[index] = value
+        try:
+            self.items[index] = value
+        except IndexError:
+            if (index == len(self.items) and value == '\x00' and
+                self._TYPE._hints.get('extra_item_after_alloc')):
+                # special case: writing NULL to the extra final char
+                return
+            raise
 
 assert not '__dict__' in dir(_array)
 assert not '__dict__' in dir(_struct)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -15,7 +15,7 @@
 from rpython.rtyper.tool.rfficache import platform, sizeof_c_type
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.annlowlevel import llhelper
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, we_are_translated_to_c
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0
 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import llmemory
@@ -232,40 +232,36 @@
             call_external_function = jit.dont_look_inside(
                 call_external_function)
 
+    def _oops():
+        raise AssertionError("can't pass (any more) a unicode string"
+                             " directly to a VOIDP argument")
+    _oops._annspecialcase_ = 'specialize:memo'
+
     unrolling_arg_tps = unrolling_iterable(enumerate(args))
     def wrapper(*args):
         real_args = ()
+        # XXX 'to_free' leaks if an allocation fails with MemoryError
+        # and was not the first in this function
         to_free = ()
         for i, TARGET in unrolling_arg_tps:
             arg = args[i]
-            freeme = None
-            if TARGET == CCHARP:
+            if TARGET == CCHARP or TARGET is VOIDP:
                 if arg is None:
                     arg = lltype.nullptr(CCHARP.TO)   # None => (char*)NULL
-                    freeme = arg
+                    to_free = to_free + (arg, '\x04')
                 elif isinstance(arg, str):
-                    arg = str2charp(arg)
-                    # XXX leaks if a str2charp() fails with MemoryError
-                    # and was not the first in this function
-                    freeme = arg
+                    tup = get_nonmovingbuffer_final_null(arg)
+                    to_free = to_free + tup
+                    arg = tup[0]
+                elif isinstance(arg, unicode):
+                    _oops()
             elif TARGET == CWCHARP:
                 if arg is None:
                     arg = lltype.nullptr(CWCHARP.TO)   # None => (wchar_t*)NULL
-                    freeme = arg
+                    to_free = to_free + (arg,)
                 elif isinstance(arg, unicode):
                     arg = unicode2wcharp(arg)
-                    # XXX leaks if a unicode2wcharp() fails with MemoryError
-                    # and was not the first in this function
-                    freeme = arg
-            elif TARGET is VOIDP:
-                if arg is None:
-                    arg = lltype.nullptr(VOIDP.TO)
-                elif isinstance(arg, str):
-                    arg = str2charp(arg)
-                    freeme = arg
-                elif isinstance(arg, unicode):
-                    arg = unicode2wcharp(arg)
-                    freeme = arg
+                    to_free = to_free + (arg,)
             elif _isfunctype(TARGET) and not _isllptr(arg):
                 # XXX pass additional arguments
                 use_gil = invoke_around_handlers
@@ -283,11 +279,22 @@
                            or TARGET is lltype.Bool)):
                         arg = cast(TARGET, arg)
             real_args = real_args + (arg,)
-            to_free = to_free + (freeme,)
         res = call_external_function(*real_args)
         for i, TARGET in unrolling_arg_tps:
-            if to_free[i]:
-                lltype.free(to_free[i], flavor='raw')
+            arg = args[i]
+            if TARGET == CCHARP or TARGET is VOIDP:
+                if arg is None:
+                    to_free = to_free[2:]
+                elif isinstance(arg, str):
+                    free_nonmovingbuffer(arg, to_free[0], to_free[1])
+                    to_free = to_free[2:]
+            elif TARGET == CWCHARP:
+                if arg is None:
+                    to_free = to_free[1:]
+                elif isinstance(arg, unicode):
+                    free_wcharp(to_free[0])
+                    to_free = to_free[1:]
+        assert len(to_free) == 0
         if rarithmetic.r_int is not r_int:
             if result is INT:
                 return cast(lltype.Signed, res)
@@ -816,52 +823,69 @@
         string is already nonmovable or could be pinned.  Must be followed by a
         free_nonmovingbuffer call.
 
-        First bool returned indicates if 'data' was pinned. Second bool returned
-        indicates if we did a raw alloc because pinning failed. Both bools
-        should never be true at the same time.
+        Also returns a char:
+         * \4: no pinning, returned pointer is inside 'data' which is nonmovable
+         * \5: 'data' was pinned, returned pointer is inside
+         * \6: pinning failed, returned pointer is raw malloced
+
+        For strings (not unicodes), the len()th character of the resulting
+        raw buffer is available, but not initialized.  Use
+        get_nonmovingbuffer_final_null() instead of get_nonmovingbuffer()
+        to get a regular null-terminated "char *".
         """
 
         lldata = llstrtype(data)
         count = len(data)
 
-        pinned = False
-        if rgc.can_move(data):
-            if rgc.pin(data):
-                pinned = True
+        if we_are_translated_to_c() and not rgc.can_move(data):
+            flag = '\x04'
+        else:
+            if we_are_translated_to_c() and rgc.pin(data):
+                flag = '\x05'
             else:
-                buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
+                buf = lltype.malloc(TYPEP.TO, count + (TYPEP is CCHARP),
+                                    flavor='raw')
                 copy_string_to_raw(lldata, buf, 0, count)
-                return buf, pinned, True
+                return buf, '\x06'
                 # ^^^ raw malloc used to get a nonmovable copy
         #
-        # following code is executed if:
+        # following code is executed after we're translated to C, if:
         # - rgc.can_move(data) and rgc.pin(data) both returned true
         # - rgc.can_move(data) returned false
         data_start = cast_ptr_to_adr(lldata) + \
             offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
 
-        return cast(TYPEP, data_start), pinned, False
+        return cast(TYPEP, data_start), flag
         # ^^^ already nonmovable. Therefore it's not raw allocated nor
         # pinned.
     get_nonmovingbuffer._always_inline_ = 'try' # get rid of the returned tuple
     get_nonmovingbuffer._annenforceargs_ = [strtype]
 
-    # (str, char*, bool, bool) -> None
+    @jit.dont_look_inside
+    def get_nonmovingbuffer_final_null(data):
+        tup = get_nonmovingbuffer(data)
+        buf, flag = tup
+        buf[len(data)] = lastchar
+        return tup
+    get_nonmovingbuffer_final_null._always_inline_ = 'try'
+    get_nonmovingbuffer_final_null._annenforceargs_ = [strtype]
+
+    # (str, char*, char) -> None
     # Can't inline this because of the raw address manipulation.
     @jit.dont_look_inside
-    def free_nonmovingbuffer(data, buf, is_pinned, is_raw):
+    def free_nonmovingbuffer(data, buf, flag):
         """
-        Keep 'data' alive and unpin it if it was pinned ('is_pinned' is true).
-        Otherwise free the non-moving copy ('is_raw' is true).
+        Keep 'data' alive and unpin it if it was pinned (flag==\5).
+        Otherwise free the non-moving copy (flag==\6).
         """
-        if is_pinned:
+        if flag == '\x05':
             rgc.unpin(data)
-        if is_raw:
+        if flag == '\x06':
             lltype.free(buf, flavor='raw')
-        # if is_pinned and is_raw are false: data was already nonmovable,
+        # if flag == '\x04': data was already nonmovable,
         # we have nothing to clean up
         keepalive_until_here(data)
-    free_nonmovingbuffer._annenforceargs_ = [strtype, None, bool, bool]
+    free_nonmovingbuffer._annenforceargs_ = [strtype, None, None]
 
     # int -> (char*, str, int)
     # Can't inline this because of the raw address manipulation.
@@ -947,18 +971,19 @@
 
     return (str2charp, free_charp, charp2str,
             get_nonmovingbuffer, free_nonmovingbuffer,
+            get_nonmovingbuffer_final_null,
             alloc_buffer, str_from_buffer, keep_buffer_alive_until_here,
             charp2strn, charpsize2str, str2chararray, str2rawmem,
             )
 
 (str2charp, free_charp, charp2str,
- get_nonmovingbuffer, free_nonmovingbuffer,
+ get_nonmovingbuffer, free_nonmovingbuffer, get_nonmovingbuffer_final_null,
  alloc_buffer, str_from_buffer, keep_buffer_alive_until_here,
  charp2strn, charpsize2str, str2chararray, str2rawmem,
  ) = make_string_mappings(str)
 
 (unicode2wcharp, free_wcharp, wcharp2unicode,
- get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer,
+ get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer, __not_usable,
  alloc_unicodebuffer, unicode_from_buffer, keep_unicodebuffer_alive_until_here,
  wcharp2unicoden, wcharpsize2unicode, unicode2wchararray, unicode2rawmem,
  ) = make_string_mappings(unicode)
@@ -1194,10 +1219,28 @@
     def __init__(self, data):
         self.data = data
     def __enter__(self):
-        self.buf, self.pinned, self.is_raw = get_nonmovingbuffer(self.data)
+        self.buf, self.flag = get_nonmovingbuffer(self.data)
         return self.buf
     def __exit__(self, *args):
-        free_nonmovingbuffer(self.data, self.buf, self.pinned, self.is_raw)
+        free_nonmovingbuffer(self.data, self.buf, self.flag)
+    __init__._always_inline_ = 'try'
+    __enter__._always_inline_ = 'try'
+    __exit__._always_inline_ = 'try'
+
+class scoped_view_charp:
+    """Returns a 'char *' that (tries to) point inside the given RPython
+    string (which must not be None).  You can replace scoped_str2charp()
+    with scoped_view_charp() in all places that guarantee that the
+    content of the 'char[]' array will not be modified.
+    """
+    def __init__(self, data):
+        self.data = data
+    __init__._annenforceargs_ = [None, annmodel.SomeString(can_be_None=False)]
+    def __enter__(self):
+        self.buf, self.flag = get_nonmovingbuffer_final_null(self.data)
+        return self.buf
+    def __exit__(self, *args):
+        free_nonmovingbuffer(self.data, self.buf, self.flag)
     __init__._always_inline_ = 'try'
     __enter__._always_inline_ = 'try'
     __exit__._always_inline_ = 'try'
@@ -1206,10 +1249,10 @@
     def __init__(self, data):
         self.data = data
     def __enter__(self):
-        self.buf, self.pinned, self.is_raw = get_nonmoving_unicodebuffer(self.data)
+        self.buf, self.flag = get_nonmoving_unicodebuffer(self.data)
         return self.buf
     def __exit__(self, *args):
-        free_nonmoving_unicodebuffer(self.data, self.buf, self.pinned, self.is_raw)
+        free_nonmoving_unicodebuffer(self.data, self.buf, self.flag)
     __init__._always_inline_ = 'try'
     __enter__._always_inline_ = 'try'
     __exit__._always_inline_ = 'try'
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -1238,7 +1238,8 @@
 # ____________________________________________________________
 
 STR.become(GcStruct('rpy_string', ('hash',  Signed),
-                    ('chars', Array(Char, hints={'immutable': True})),
+                    ('chars', Array(Char, hints={'immutable': True,
+                                    'extra_item_after_alloc': 1})),
                     adtmeths={'malloc' : staticAdtMethod(mallocstr),
                               'empty'  : staticAdtMethod(emptystrfun),
                               'copy_contents' : staticAdtMethod(copy_string_contents),
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -516,7 +516,7 @@
     def test_nonmovingbuffer(self):
         d = 'some cool data that should not move'
         def f():
-            buf, is_pinned, is_raw = get_nonmovingbuffer(d)
+            buf, flag = get_nonmovingbuffer(d)
             try:
                 counter = 0
                 for i in range(len(d)):
@@ -524,7 +524,7 @@
                         counter += 1
                 return counter
             finally:
-                free_nonmovingbuffer(d, buf, is_pinned, is_raw)
+                free_nonmovingbuffer(d, buf, flag)
         assert f() == len(d)
         fn = self.compile(f, [], gcpolicy='ref')
         assert fn() == len(d)
@@ -534,13 +534,13 @@
         def f():
             counter = 0
             for n in range(32):
-                buf, is_pinned, is_raw = get_nonmovingbuffer(d)
+                buf, flag = get_nonmovingbuffer(d)
                 try:
                     for i in range(len(d)):
                         if buf[i] == d[i]:
                             counter += 1
                 finally:
-                    free_nonmovingbuffer(d, buf, is_pinned, is_raw)
+                    free_nonmovingbuffer(d, buf, flag)
             return counter
         fn = self.compile(f, [], gcpolicy='semispace')
         # The semispace gc uses raw_malloc for its internal data structs
@@ -555,13 +555,13 @@
         def f():
             counter = 0
             for n in range(32):
-                buf, is_pinned, is_raw = get_nonmovingbuffer(d)
+                buf, flag = get_nonmovingbuffer(d)
                 try:
                     for i in range(len(d)):
                         if buf[i] == d[i]:
                             counter += 1
                 finally:
-                    free_nonmovingbuffer(d, buf, is_pinned, is_raw)
+                    free_nonmovingbuffer(d, buf, flag)
             return counter
         fn = self.compile(f, [], gcpolicy='incminimark')
         # The incminimark gc uses raw_malloc for its internal data structs
@@ -835,3 +835,11 @@
     if hasattr(rffi, '__INT128_T'):
         value = 0xAAAABBBBCCCCDDDD
         assert cast(rffi.__INT128_T, r_uint64(value)) == value
+
+def test_scoped_view_charp():
+    s = 'bar'
+    with scoped_view_charp(s) as buf:
+        assert buf[0] == 'b'
+        assert buf[1] == 'a'
+        assert buf[2] == 'r'
+        assert buf[3] == '\x00'
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -253,8 +253,11 @@
             yield '\t' + cdecl(typename, fname) + ';'
         if not self.ARRAY._hints.get('nolength', False):
             yield '\tlong length;'
+        varlength = self.varlength
+        if varlength is not None:
+            varlength += self.ARRAY._hints.get('extra_item_after_alloc', 0)
         line = '%s;' % cdecl(self.itemtypename,
-                             'items[%s]' % deflength(self.varlength))
+                             'items[%s]' % deflength(varlength))
         if self.ARRAY.OF is Void:    # strange
             line = '/* array of void */'
             if self.ARRAY._hints.get('nolength', False):
diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py
--- a/rpython/translator/c/test/test_lltyped.py
+++ b/rpython/translator/c/test/test_lltyped.py
@@ -1,4 +1,4 @@
-import py
+import py, random
 from rpython.rtyper.lltypesystem.lltype import *
 from rpython.rtyper.lltypesystem import rffi
 from rpython.translator.c.test.test_genc import compile
@@ -255,28 +255,6 @@
         res2 = fn(0)
         assert res1 == res2
 
-    def test_null_padding(self):
-        py.test.skip("we no longer pad our RPython strings with a final NUL")
-        from rpython.rtyper.lltypesystem import llmemory
-        from rpython.rtyper.lltypesystem import rstr
-        chars_offset = llmemory.FieldOffset(rstr.STR, 'chars') + \
-                       llmemory.ArrayItemsOffset(rstr.STR.chars)
-        # sadly, there's no way of forcing this to fail if the strings
-        # are allocated in a region of memory such that they just
-        # happen to get a NUL byte anyway :/ (a debug build will
-        # always fail though)
-        def trailing_byte(s):
-            adr_s = llmemory.cast_ptr_to_adr(s)
-            return (adr_s + chars_offset).char[len(s)]
-        def f(x):
-            r = 0
-            for i in range(x):
-                r += ord(trailing_byte(' '*(100-x*x)))
-            return r
-        fn = self.getcompiled(f, [int])
-        res = fn(10)
-        assert res == 0
-
     def test_cast_primitive(self):
         def f(x):
             x = cast_primitive(UnsignedLongLong, x)
@@ -1023,3 +1001,49 @@
         assert fn(r_longlong(1)) == True
         assert fn(r_longlong(256)) == True
         assert fn(r_longlong(2**32)) == True
+
+    def test_extra_item_after_alloc(self):
+        from rpython.rlib import rgc
+        from rpython.rtyper.lltypesystem import lltype
+        from rpython.rtyper.lltypesystem import rstr
+        # all STR objects should be allocated with enough space for one
+        # extra char.  Check this for prebuilt strings, and for dynamically
+        # allocated ones with the default GC for tests.  Use strings of 8,
+        # 16 and 24 chars because if the extra char is missing, writing to it
+        # is likely to cause corruption in nearby structures.
+        sizes = [random.choice([8, 16, 24]) for i in range(100)]
+        A = lltype.Struct('A', ('x', lltype.Signed))
+        prebuilt = [(rstr.mallocstr(sz),
+                     lltype.malloc(A, flavor='raw', immortal=True))
+                        for sz in sizes]
+        k = 0
+        for i, (s, a) in enumerate(prebuilt):
+            a.x = i
+            for i in range(len(s.chars)):
+                k += 1
+                if k == 256:
+                    k = 1
+                s.chars[i] = chr(k)
+
+        def check(lst):
+            hashes = []
+            for i, (s, a) in enumerate(lst):
+                assert a.x == i
+                rgc.ll_write_final_null_char(s)
+            for i, (s, a) in enumerate(lst):
+                assert a.x == i     # check it was not overwritten
+        def f():
+            check(prebuilt)
+            lst1 = []
+            for i, sz in enumerate(sizes):
+                s = rstr.mallocstr(sz)
+                a = lltype.malloc(A, flavor='raw')
+                a.x = i
+                lst1.append((s, a))
+            check(lst1)
+            for _, a in lst1:
+                lltype.free(a, flavor='raw')
+            return 42
+
+        fn = self.getcompiled(f, [])
+        assert fn() == 42
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -3,6 +3,7 @@
 import os
 import sys
 import subprocess
+import random
 
 import py
 
@@ -1468,6 +1469,52 @@
         res = self.run('nursery_hash_base')
         assert res >= 195
 
+    def define_extra_item_after_alloc(cls):
+        from rpython.rtyper.lltypesystem import rstr
+        # all STR objects should be allocated with enough space for
+        # one extra char.  Check this with our GCs.  Use strings of 8,
+        # 16 and 24 chars because if the extra char is missing,
+        # writing to it is likely to cause corruption in nearby
+        # structures.
+        sizes = [random.choice([8, 16, 24]) for i in range(100)]
+        A = lltype.Struct('A', ('x', lltype.Signed))
+        prebuilt = [(rstr.mallocstr(sz),
+                     lltype.malloc(A, flavor='raw', immortal=True))
+                        for sz in sizes]
+        k = 0
+        for i, (s, a) in enumerate(prebuilt):
+            a.x = i
+            for i in range(len(s.chars)):
+                k += 1
+                if k == 256:
+                    k = 1
+                s.chars[i] = chr(k)
+
+        def check(lst):
+            hashes = []
+            for i, (s, a) in enumerate(lst):
+                assert a.x == i
+                rgc.ll_write_final_null_char(s)
+            for i, (s, a) in enumerate(lst):
+                assert a.x == i     # check it was not overwritten
+        def fn():
+            check(prebuilt)
+            lst1 = []
+            for i, sz in enumerate(sizes):
+                s = rstr.mallocstr(sz)
+                a = lltype.malloc(A, flavor='raw')
+                a.x = i
+                lst1.append((s, a))
+            check(lst1)
+            for _, a in lst1:
+                lltype.free(a, flavor='raw')
+            return 42
+        return fn
+
+    def test_extra_item_after_alloc(self):
+        res = self.run('extra_item_after_alloc')
+        assert res == 42
+
 
 class TestGenerationalGC(TestSemiSpaceGC):
     gcpolicy = "generation"
diff --git a/rpython/translator/tool/test/test_staticsizereport.py b/rpython/translator/tool/test/test_staticsizereport.py
--- a/rpython/translator/tool/test/test_staticsizereport.py
+++ b/rpython/translator/tool/test/test_staticsizereport.py
@@ -67,7 +67,7 @@
             (4 * S + 2 * P) +      # struct dicttable
             (S + 2 * 8192) +       # indexes, length 8192, rffi.USHORT
             (S + (S + S) * 3840) + # entries, length 3840
-            (S + S + 5) * 3840)    # 3840 strings with 5 chars each
+            (S + S + 6) * 3840)    # 3840 strings with 5 chars each (+1 final)
         assert guess_size(func.builder.db, fixarrayvalnode, set()) == 100 * rffi.sizeof(lltype.Signed) + 1 * rffi.sizeof(lltype.Signed)
         assert guess_size(func.builder.db, dynarrayvalnode, set()) == 100 * rffi.sizeof(lltype.Signed) + 2 * rffi.sizeof(lltype.Signed) + 1 * rffi.sizeof(rffi.VOIDP)
 

From pypy.commits at gmail.com  Tue Aug  2 12:11:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 09:11:42 -0700 (PDT)
Subject: [pypy-commit] cffi default: The null_byte_after_str branch of PyPy
 makes ffi.from_buffer(str)
Message-ID: <57a0c63e.65efc20a.6e316.2043@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2731:a54b242f428f
Date: 2016-08-02 18:13 +0200
http://bitbucket.org/cffi/cffi/changeset/a54b242f428f/

Log:	The null_byte_after_str branch of PyPy makes ffi.from_buffer(str)
	possible.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5974,6 +5974,11 @@
 
 static int invalid_input_buffer_type(PyObject *x)
 {
+    /* From PyPy 5.4, from_buffer() accepts strings (but still not buffers
+       or memoryviews on strings). */
+    if (PyBytes_Check(x))
+        return 0;
+
 #if PY_MAJOR_VERSION < 3
     if (PyBuffer_Check(x)) {
         /* XXX fish fish fish in an inofficial way */
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3341,13 +3341,18 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharA = new_array_type(BCharP, None)
-    py.test.raises(TypeError, from_buffer, BCharA, b"foo")
+    p1 = from_buffer(BCharA, b"foo")
+    assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # from_buffer(buffer(b"foo")) does not work, because it's not
+        # implemented on pypy; only from_buffer(b"foo") works.
         py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
         py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
     try:
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -171,7 +171,7 @@
 buffer interface.  This is the opposite of ``ffi.buffer()``.  It gives
 a reference to the existing data, not a copy; for this
 reason, and for PyPy compatibility, it does not work with the built-in
-types str or unicode (or buffers/memoryviews on them).
+type unicode; nor buffers/memoryviews to byte or unicode strings.
 It is meant to be used on objects
 containing large quantities of raw data, like bytearrays
 or ``array.array`` or numpy
@@ -193,6 +193,9 @@
 method is called), then the ``<cdata>`` object will point to freed
 memory and must not be used any more.
 
+*New in version 1.8:* the python_buffer can be a byte string (but still
+not a buffer/memoryview on a string).
+
 
 ffi.memmove()
 +++++++++++++
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,13 @@
 ======================
 
 
+v1.8
+====
+
+* Removed the restriction that ``ffi.from_buffer()`` cannot be used on
+  byte strings (PyPy was improved and can now support that case).
+
+
 v1.7
 ====
 

From pypy.commits at gmail.com  Tue Aug  2 12:13:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 09:13:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document branch
Message-ID: <57a0c696.82ddc20a.5e0f1.23d5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85992:7d9e19e2e836
Date: 2016-08-02 18:15 +0200
http://bitbucket.org/pypy/pypy/changeset/7d9e19e2e836/

Log:	Document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -109,3 +109,11 @@
 .. branch: jitlog-exact-source-lines
 
 Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)``.

From pypy.commits at gmail.com  Tue Aug  2 13:10:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 02 Aug 2016 10:10:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: More documentation
Message-ID: <57a0d413.041f1c0a.8b27f.f274@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r85993:fe9b1fb401fa
Date: 2016-08-02 19:12 +0200
http://bitbucket.org/pypy/pypy/changeset/fe9b1fb401fa/

Log:	More documentation

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -116,4 +116,6 @@
 It is used to hold a final zero in case we need some ``char *``
 representation of the string, together with checks like ``not
 can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)``.
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.

From pypy.commits at gmail.com  Tue Aug  2 15:08:23 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 Aug 2016 12:08:23 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Create a real dict using the
 ClassDictStrategy in W_TypeObject.getdict()
Message-ID: <57a0efa7.c4ebc20a.86c3a.5a66@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r85994:b7e537a214fd
Date: 2016-08-02 20:01 +0100
http://bitbucket.org/pypy/pypy/changeset/b7e537a214fd/

Log:	Create a real dict using the ClassDictStrategy in
	W_TypeObject.getdict()

	* DictProxyStrategy renamed to ClassDictStrategy
	* dictproxyobject.py renamed to classdict.py

	Note: in CPython, type.__dict__ returns a hidden mutable dict
	opaquely wrapped by a read-only mappingproxy object. This commit
	creates the former.

diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/classdict.py
rename from pypy/objspace/std/dictproxyobject.py
rename to pypy/objspace/std/classdict.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/classdict.py
@@ -38,7 +38,7 @@
 )
 
 
-class DictProxyStrategy(DictStrategy):
+class ClassDictStrategy(DictStrategy):
     """Exposes a W_TypeObject.dict_w at app-level.
 
     Uses getdictvalue() and setdictvalue() to access items.
@@ -142,7 +142,7 @@
     # keys are utf-8 encoded identifiers from type's dict_w
     return space.wrap(key.decode('utf-8'))
 
-create_iterator_classes(DictProxyStrategy)
+create_iterator_classes(ClassDictStrategy)
 
 
 class MappingProxyStrategy(DictStrategy):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -484,13 +484,13 @@
             del self.lazyloaders
 
     def getdict(self, space): # returning a dict-proxy!
-        from pypy.objspace.std.dictproxyobject import DictProxyStrategy
-        from pypy.objspace.std.dictproxyobject import W_DictProxyObject
+        from pypy.objspace.std.classdict import ClassDictStrategy
+        from pypy.objspace.std.dictmultiobject import W_DictObject
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        strategy = space.fromcache(DictProxyStrategy)
+        strategy = space.fromcache(ClassDictStrategy)
         storage = strategy.erase(self)
-        return W_DictProxyObject(space, strategy, storage)
+        return W_DictObject(space, strategy, storage)
 
     def is_heaptype(self):
         return self.flag_heaptype

From pypy.commits at gmail.com  Tue Aug  2 15:51:39 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 02 Aug 2016 12:51:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Add parser and syntax tests for
 async and await, remove set_sentinel comment
Message-ID: <57a0f9cb.271ac20a.e6d8c.6887@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r85995:cf642e39ca7f
Date: 2016-08-02 21:51 +0200
http://bitbucket.org/pypy/pypy/changeset/cf642e39ca7f/

Log:	Add parser and syntax tests for async and await, remove set_sentinel
	comment

diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -169,8 +169,15 @@
         py.test.raises(SyntaxError, self.parse, 'x = 5 # comment\nx = 6\n', "single")
 
     def test_async_await(self):
-        self.parse("async def coro(): pass")
-        self.parse("await result = func()")
+        self.parse("async def coro(): await func")
+        #Test as var and func name
+        self.parse("async = 1")
+        self.parse("await = 1")
+        self.parse("def async(): pass")
+        #async for
+        self.parse("async def foo(): async for a in b: pass")
+        #async with
+        self.parse("async def foo(): async with a: pass")
 
 class TestPythonParserWithSpace:
 
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -99,7 +99,13 @@
     
     async def foo():
         await await fut
-
+    
+    await x
+    
+    def foo(): async for a in b: pass
+    
+    def foo(): async with a: pass
+    
 """)
 
 
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -147,7 +147,6 @@
 def set_sentinel(space):
     """Set a sentinel lock that will be released when the current thread
 state is finalized (after it is untied from the interpreter)."""
-    #NOT IMPLEMENTED YET!!! (required for some libs to work)
     return space.wrap(Lock(space))
 
 class W_RLock(W_Root):

From pypy.commits at gmail.com  Tue Aug  2 18:03:17 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 Aug 2016 15:03:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merged in
 marky1991/pypy_new/py3k-finish_time (pull request #465)
Message-ID: <57a118a5.10a81c0a.7b92a.554d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r85997:b5091dc062fd
Date: 2016-08-02 23:02 +0100
http://bitbucket.org/pypy/pypy/changeset/b5091dc062fd/

Log:	Merged in marky1991/pypy_new/py3k-finish_time (pull request #465)

	Make all tests pass for the time module.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -541,6 +541,8 @@
         t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
         t_ref[0] = tt
         pbuf = c_localtime(t_ref)
+        rffi.setintfield(pbuf, "c_tm_year",
+                         rffi.getintfield(pbuf, "c_tm_year") + 1900)
         lltype.free(t_ref, flavor='raw')
         if not pbuf:
             raise OperationError(space.w_ValueError,
@@ -584,7 +586,7 @@
     if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
         raise oefmt(space.w_ValueError, "day of week out of range")
 
-    rffi.setintfield(glob_buf, 'c_tm_year', y - 1900)
+    rffi.setintfield(glob_buf, 'c_tm_year', y)
     rffi.setintfield(glob_buf, 'c_tm_mon',
                      rffi.getintfield(glob_buf, 'c_tm_mon') - 1)
     rffi.setintfield(glob_buf, 'c_tm_wday',
@@ -648,7 +650,8 @@
         t_ref[0] = seconds
         p = c_localtime(t_ref)
     if not p:
-        raise oefmt(space.w_ValueError, "unconvertible time")
+        raise oefmt(space.w_OSError, "unconvertible time")
+    rffi.setintfield(p, "c_tm_year", rffi.getintfield(p, "c_tm_year") + 1900)
     return _asctime(space, p)
 
 # by now w_tup is an optional argument (and not *args)
@@ -677,7 +680,7 @@
             w(getif(t_ref, 'c_tm_hour')),
             w(getif(t_ref, 'c_tm_min')),
             w(getif(t_ref, 'c_tm_sec')),
-            w(getif(t_ref, 'c_tm_year') + 1900)]
+            w(getif(t_ref, 'c_tm_year'))]
     return space.mod(w("%.3s %.3s%3d %.2d:%.2d:%.2d %d"),
                      space.newtuple(args))
 
@@ -715,7 +718,7 @@
     lltype.free(t_ref, flavor='raw')
 
     if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
+        raise OperationError(space.w_OSError, space.wrap(_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def mktime(space, w_tup):
@@ -725,6 +728,7 @@
 
     buf = _gettmarg(space, w_tup, allowNone=False)
     rffi.setintfield(buf, "c_tm_wday", -1)
+    rffi.setintfield(buf, "c_tm_year", rffi.getintfield(buf, "c_tm_year") - 1900)
     tt = c_mktime(buf)
     # A return value of -1 does not necessarily mean an error, but tm_wday
     # cannot remain set to -1 if mktime succeeds.
@@ -801,6 +805,8 @@
         rffi.setintfield(buf_value, 'c_tm_isdst', -1)
     elif rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
         rffi.setintfield(buf_value, 'c_tm_isdst', 1)
+    rffi.setintfield(buf_value, "c_tm_year",
+                     rffi.getintfield(buf_value, "c_tm_year") - 1900)
 
     if _WIN:
         # check that the format string contains only valid directives

From pypy.commits at gmail.com  Tue Aug  2 18:03:28 2016
From: pypy.commits at gmail.com (marky1991)
Date: Tue, 02 Aug 2016 15:03:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-finish_time: Make all tests pass for the
 time module. lib-python/test_time.py still fails test_mktime_error when run
 untranslated, but this test is skipped when run translated anyway.
Message-ID: <57a118b0.28eac20a.8f95a.922a@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k-finish_time
Changeset: r85996:80df0bb39be6
Date: 2016-07-17 23:51 -0400
http://bitbucket.org/pypy/pypy/changeset/80df0bb39be6/

Log:	Make all tests pass for the time module. lib-python/test_time.py
	still fails test_mktime_error when run untranslated, but this test
	is skipped when run translated anyway.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -541,6 +541,8 @@
         t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
         t_ref[0] = tt
         pbuf = c_localtime(t_ref)
+        rffi.setintfield(pbuf, "c_tm_year",
+                         rffi.getintfield(pbuf, "c_tm_year") + 1900)
         lltype.free(t_ref, flavor='raw')
         if not pbuf:
             raise OperationError(space.w_ValueError,
@@ -584,7 +586,7 @@
     if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
         raise oefmt(space.w_ValueError, "day of week out of range")
 
-    rffi.setintfield(glob_buf, 'c_tm_year', y - 1900)
+    rffi.setintfield(glob_buf, 'c_tm_year', y)
     rffi.setintfield(glob_buf, 'c_tm_mon',
                      rffi.getintfield(glob_buf, 'c_tm_mon') - 1)
     rffi.setintfield(glob_buf, 'c_tm_wday',
@@ -648,7 +650,8 @@
         t_ref[0] = seconds
         p = c_localtime(t_ref)
     if not p:
-        raise oefmt(space.w_ValueError, "unconvertible time")
+        raise oefmt(space.w_OSError, "unconvertible time")
+    rffi.setintfield(p, "c_tm_year", rffi.getintfield(p, "c_tm_year") + 1900)
     return _asctime(space, p)
 
 # by now w_tup is an optional argument (and not *args)
@@ -677,7 +680,7 @@
             w(getif(t_ref, 'c_tm_hour')),
             w(getif(t_ref, 'c_tm_min')),
             w(getif(t_ref, 'c_tm_sec')),
-            w(getif(t_ref, 'c_tm_year') + 1900)]
+            w(getif(t_ref, 'c_tm_year'))]
     return space.mod(w("%.3s %.3s%3d %.2d:%.2d:%.2d %d"),
                      space.newtuple(args))
 
@@ -715,7 +718,7 @@
     lltype.free(t_ref, flavor='raw')
 
     if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
+        raise OperationError(space.w_OSError, space.wrap(_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def mktime(space, w_tup):
@@ -725,6 +728,7 @@
 
     buf = _gettmarg(space, w_tup, allowNone=False)
     rffi.setintfield(buf, "c_tm_wday", -1)
+    rffi.setintfield(buf, "c_tm_year", rffi.getintfield(buf, "c_tm_year") - 1900)
     tt = c_mktime(buf)
     # A return value of -1 does not necessarily mean an error, but tm_wday
     # cannot remain set to -1 if mktime succeeds.
@@ -801,6 +805,8 @@
         rffi.setintfield(buf_value, 'c_tm_isdst', -1)
     elif rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
         rffi.setintfield(buf_value, 'c_tm_isdst', 1)
+    rffi.setintfield(buf_value, "c_tm_year",
+                     rffi.getintfield(buf_value, "c_tm_year") - 1900)
 
     if _WIN:
         # check that the format string contains only valid directives
@@ -970,8 +976,11 @@
              lltype.scoped_alloc(rwin32.FILETIME) as exit_time, \
              lltype.scoped_alloc(rwin32.FILETIME) as kernel_time, \
              lltype.scoped_alloc(rwin32.FILETIME) as user_time:
-            GetProcessTimes(current_process, creation_time, exit_time,
-                            kernel_time, user_time)
+            worked = GetProcessTimes(current_process, creation_time, exit_time,
+                                     kernel_time, user_time)
+            if not worked:
+                raise wrap_windowserror(space,
+                    rwin32.lastSavedWindowsError("GetProcessTimes"))
             kernel_time2 = (kernel_time.c_dwLowDateTime |
                             r_ulonglong(kernel_time.c_dwHighDateTime) << 32)
             user_time2 = (user_time.c_dwLowDateTime |

From pypy.commits at gmail.com  Wed Aug  3 00:29:21 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 Aug 2016 21:29:21 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Fix test_dictproxy.py to actually
 match the expected behaviour
Message-ID: <57a17321.e129c20a.19a07.e965@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r85998:25dc55060e7c
Date: 2016-08-03 04:59 +0100
http://bitbucket.org/pypy/pypy/changeset/25dc55060e7c/

Log:	Fix test_dictproxy.py to actually match the expected behaviour

diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -9,42 +9,20 @@
         assert 'a' in NotEmpty.__dict__
         assert 'a' in NotEmpty.__dict__.keys()
         assert 'b' not in NotEmpty.__dict__
-        NotEmpty.__dict__['b'] = 4
-        assert NotEmpty.b == 4
-        del NotEmpty.__dict__['b']
         assert NotEmpty.__dict__.get("b") is None
+        raises(TypeError, "NotEmpty.__dict__['b'] = 4")
         raises(TypeError, 'NotEmpty.__dict__[15] = "y"')
-        raises(KeyError, 'del NotEmpty.__dict__[15]')
+        raises(TypeError, 'del NotEmpty.__dict__[15]')
 
-        assert NotEmpty.__dict__.setdefault("string", 1) == 1
-        assert NotEmpty.__dict__.setdefault("string", 2) == 1
-        assert NotEmpty.string == 1
-        raises(TypeError, 'NotEmpty.__dict__.setdefault(15, 1)')
-
-    def test_dictproxy_popitem(self):
-        class A(object):
-            a = 42
-        seen = 0
-        try:
-            while True:
-                key, value = A.__dict__.popitem()
-                if key == 'a':
-                    assert value == 42
-                    seen += 1
-        except KeyError:
-            pass
-        assert seen == 1
+        raises(AttributeError, 'NotEmpty.__dict__.setdefault')
 
     def test_dictproxy_getitem(self):
         class NotEmpty(object):
             a = 1
         assert 'a' in NotEmpty.__dict__
-        class substr(str): pass
+        class substr(str):
+            pass
         assert substr('a') in NotEmpty.__dict__
-        # the following are only for py2
-        ## assert u'a' in NotEmpty.__dict__
-        ## assert NotEmpty.__dict__[u'a'] == 1
-        ## assert u'\xe9' not in NotEmpty.__dict__
 
     def test_dictproxyeq(self):
         class a(object):
@@ -63,9 +41,9 @@
         class a(object):
             pass
         s1 = repr(a.__dict__)
+        assert s1.startswith('mappingproxy({') and s1.endswith('})')
         s2 = str(a.__dict__)
-        assert s1 == s2
-        assert s1.startswith('mappingproxy({') and s1.endswith('})')
+        assert s1 == 'mappingproxy(%s)' % s2
 
     def test_immutable_dict_on_builtin_type(self):
         raises(TypeError, "int.__dict__['a'] = 1")
@@ -100,4 +78,3 @@
 
 class AppTestUserObjectMethodCache(AppTestUserObject):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
-

From pypy.commits at gmail.com  Wed Aug  3 00:29:23 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 Aug 2016 21:29:23 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Move cpyext implementation of
 mappingproxy to objspace
Message-ID: <57a17323.109a1c0a.ee027.0f1a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r85999:fd9153375a2a
Date: 2016-08-03 05:17 +0100
http://bitbucket.org/pypy/pypy/changeset/fd9153375a2a/

Log:	Move cpyext implementation of mappingproxy to objspace

diff --git a/pypy/module/cpyext/dictproxyobject.py b/pypy/module/cpyext/dictproxyobject.py
--- a/pypy/module/cpyext/dictproxyobject.py
+++ b/pypy/module/cpyext/dictproxyobject.py
@@ -1,67 +1,7 @@
-# Read-only proxy for mappings. PyPy does not have a separate type for
-# type.__dict__, so PyDictProxy_New has to use a custom read-only mapping.
-
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.interpreter.typedef import TypeDef, interp2app
+from pypy.objspace.std.dictproxyobject import W_DictProxyObject
 from pypy.module.cpyext.api import cpython_api, build_type_checkers
 from pypy.module.cpyext.pyobject import PyObject
 
-class W_DictProxyObject(W_Root):
-    "Read-only proxy for mappings."
-
-    def __init__(self, w_mapping):
-        self.w_mapping = w_mapping
-
-    def descr_len(self, space):
-        return space.len(self.w_mapping)
-
-    def descr_getitem(self, space, w_key):
-        return space.getitem(self.w_mapping, w_key)
-
-    def descr_contains(self, space, w_key):
-        return space.contains(self.w_mapping, w_key)
-
-    def descr_iter(self, space):
-        return space.iter(self.w_mapping)
-
-    def descr_str(self, space):
-        return space.str(self.w_mapping)
-
-    def descr_repr(self, space):
-        return space.repr(self.w_mapping)
-
-    @unwrap_spec(w_default=WrappedDefault(None))
-    def get_w(self, space, w_key, w_default):
-        return space.call_method(self.w_mapping, "get", w_key, w_default)
-
-    def keys_w(self, space):
-        return space.call_method(self.w_mapping, "keys")
-
-    def values_w(self, space):
-        return space.call_method(self.w_mapping, "values")
-
-    def items_w(self, space):
-        return space.call_method(self.w_mapping, "items")
-
-    def copy_w(self, space):
-        return space.call_method(self.w_mapping, "copy")
-
-W_DictProxyObject.typedef = TypeDef(
-    'mappingproxy',
-    __len__=interp2app(W_DictProxyObject.descr_len),
-    __getitem__=interp2app(W_DictProxyObject.descr_getitem),
-    __contains__=interp2app(W_DictProxyObject.descr_contains),
-    __iter__=interp2app(W_DictProxyObject.descr_iter),
-    __str__=interp2app(W_DictProxyObject.descr_str),
-    __repr__=interp2app(W_DictProxyObject.descr_repr),
-    get=interp2app(W_DictProxyObject.get_w),
-    keys=interp2app(W_DictProxyObject.keys_w),
-    values=interp2app(W_DictProxyObject.values_w),
-    items=interp2app(W_DictProxyObject.items_w),
-    copy=interp2app(W_DictProxyObject.copy_w)
-)
-
 PyDictProxy_Check, PyDictProxy_CheckExact = build_type_checkers(
     "DictProxy", W_DictProxyObject)
 
diff --git a/pypy/module/cpyext/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
copy from pypy/module/cpyext/dictproxyobject.py
copy to pypy/objspace/std/dictproxyobject.py
--- a/pypy/module/cpyext/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -4,8 +4,6 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.interpreter.typedef import TypeDef, interp2app
-from pypy.module.cpyext.api import cpython_api, build_type_checkers
-from pypy.module.cpyext.pyobject import PyObject
 
 class W_DictProxyObject(W_Root):
     "Read-only proxy for mappings."
@@ -61,10 +59,3 @@
     items=interp2app(W_DictProxyObject.items_w),
     copy=interp2app(W_DictProxyObject.copy_w)
 )
-
-PyDictProxy_Check, PyDictProxy_CheckExact = build_type_checkers(
-    "DictProxy", W_DictProxyObject)
-
- at cpython_api([PyObject], PyObject)
-def PyDictProxy_New(space, w_dict):
-    return space.wrap(W_DictProxyObject(w_dict))
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -486,11 +486,13 @@
     def getdict(self, space): # returning a dict-proxy!
         from pypy.objspace.std.classdict import ClassDictStrategy
         from pypy.objspace.std.dictmultiobject import W_DictObject
+        from pypy.objspace.std.dictproxyobject import W_DictProxyObject
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
         strategy = space.fromcache(ClassDictStrategy)
         storage = strategy.erase(self)
-        return W_DictObject(space, strategy, storage)
+        w_dict = W_DictObject(space, strategy, storage)
+        return W_DictProxyObject(w_dict)
 
     def is_heaptype(self):
         return self.flag_heaptype

From pypy.commits at gmail.com  Wed Aug  3 04:55:15 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 03 Aug 2016 01:55:15 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: updates to the blog post draft
 (addressed cfbolz's comments)
Message-ID: <57a1b173.43681c0a.2325f.5b4b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5661:37e031d5a693
Date: 2016-08-02 18:56 +0200
http://bitbucket.org/pypy/extradoc/changeset/37e031d5a693/

Log:	updates to the blog post draft (addressed cfbolz's comments)

diff --git a/blog/draft/new-jit-log.rst b/blog/draft/new-jit-log.rst
--- a/blog/draft/new-jit-log.rst
+++ b/blog/draft/new-jit-log.rst
@@ -1,18 +1,26 @@
-PyPy's Toolbox got a new Hammer &#x1f528;
+JitViewer moves to vmprof.com
 =======
 
-.. : XXX the title is very generic
+We are happy to announce that VMProf got a major update. The most significant change is the movement of JitViewer (JV)
+to VMProf.
+JV allows you to inspect PyPy's internal compiler representation including the generated machine code of your program.
+A useful tool to understand PyPy, learn many details of our compiler and find potential issues related to our JIT.
+Both VMProf and JV share some common goals. That is the reason why they are now both packaged together.
+www.vmprof.com also got updated with various bugfixes and changes including an all new interface to JV.
 
-.. : XXX I don't actually like the first paragraph, I think it should be more
-    to the point. eg that things happened at the Leysin sprint doesn't matter much.
-    I would also add links to all the existing tools
+A advertisment: We constantly improve tooling and libraries around the Python/PyPy eco system.
+Here are a three examples you might also want to use in your Python projects:
 
-Tools, tools, tools! It seems that PyPy cannot get enough of them!
-In the last winter sprint (Leysin) covered the current tool for observing internals of the JIT compiler (JitViewer). VMProf at that time already proved that it is a good tool for CPU profiles. We are happy to release a new version of VMProf incorporating a rewritten version of JitViewer.
+* VMProf - A statistical CPU profiler
+* RevDB - A reverse debugger for Python 
+* CFFI - Foreign Function Interface that avoids CPyExt
 
-The old logging format was a hard to maintain plain text logging facility. Frequent changes often broke internal tools, most notably the JITViewer. Another problem was that the logging output of a long running program took a lot of disk space.
+A brand new JitViewer
+---------------------
 
-Our new binary format encodes data densly, makes use of some compression (gzip) and tries to remove repetition where possible. On top of that protocol supports versioning and can be extended easily. And *drumroll* you do not need to install JitViewer yourself anymore! The whole system moved to vmprof.com and you can use it any time.
+The old logging format was a hard to maintain plain text logging facility. Frequent changes often broke internal tools, most notably JV. Additionaly the logging output of a long running program took a lot of disk space.
+
+Our new binary format encodes data densly, makes use of some compression (gzip) and tries to remove repetition where possible. On top of that protocol supports versioning and can be extended easily. And *drumroll* you do not need to install JV yourself anymore! The whole system moved to vmprof.com and you can use it any time.
 
 Sounds great. But what can you do with it? Here are two examples useful for a PyPy user:
 
@@ -21,19 +29,19 @@
 
 For some hard to find bugs it is often necessary to look at the compiled code. The old procedure often required to upload a plain text file which was hard to parse and to look through. 
 
-The new way to share a crash report is to install the ``vmprof`` module from PyPi and execute either of the two commands:
+A new way to share a crash report is to install the ``vmprof`` module from PyPi and execute either of the two commands:
 
 ```
 # this program does not crash, but has some weird behaviour
 $ pypy -m jitlog --web <your program args>
 ...
-PyPy Jitlog: http://vmprof.com/#/<hash>
+PyPy Jitlog: http://vmprof.com/#/<hash>/traces
 # this program segfaults
 $ pypy -m jitlog -o /tmp/log <your program args>
 ...
 <Segfault>
 $ pypy -m jitlog --upload /tmp/log
-PyPy Jitlog: http://vmprof.com/#/<hash>
+PyPy Jitlog: http://vmprof.com/#/<hash>/traces
 ```
 
 Providing the link in the bug report enables PyPy developers browse and identify potential issues.
@@ -41,7 +49,7 @@
 Speed issues
 ------------
 
-VMProf is a great tool to find out hot spots that consume a lot of time in your program. As soon as you have idenified code that runs slow, you can switch to jitlog and maybe pin point certain aspects that do not behave as expected. You will find not only the overview, but are also able to browse the generated code. If you cannot make sense of that all you can just share the link with us and we can have a look at the compiled code.
+VMProf is a great tool to find out hot spots that consume a lot of time in your program. As soon as you have idenified code that runs slow, you can switch to jitlog and maybe pin point certain aspects that do not behave as expected. You will find not only the overview, but are also able to browse the generated code. If you cannot make sense of that all you can just share the link with us and we can have a look at too.
 
 Future direction
 ----------------
@@ -52,7 +60,7 @@
 
 * Combination of CPU profiles and the JITLOG (Sadly did not make it into the current release)
 
-* Extend vmprof.com to be able to query vmprof/jitlog. Some times it is interesting to search for specific patterns the compiler produced. An example for vmprof: 'methods.callsites() > 5' and for the jitlog would be 'traces.contains('call_assembler').hasbridge('*my_func_name*')'
+* Extend vmprof.com to be able to query vmprof/jitlog. An example query for vmprof: 'methods.callsites() > 5' and for the jitlog would be 'traces.contains('call_assembler').hasbridge('*my_func_name*')'.
 
 * Extend the jitlog to capture the information of the optimization stage
 

From pypy.commits at gmail.com  Wed Aug  3 07:34:42 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 03 Aug 2016 04:34:42 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57a1d6d2.56421c0a.75916.9d16@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86000:8ec827c25980
Date: 2016-08-02 09:29 +0200
http://bitbucket.org/pypy/pypy/changeset/8ec827c25980/

Log:	merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -101,3 +101,11 @@
 .. branch: jitlog-32bit
 
 Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -1,8 +1,9 @@
-
 import sys
+import platform
 from rpython.tool.udir import udir
 from pypy.tool.pytest.objspace import gettestobjspace
 from rpython.rlib.rjitlog import rjitlog as jl
+from rpython.jit.metainterp.resoperation import opname
 
 class AppTestJitLog(object):
     spaceconfig = {'usemodules': ['_jitlog', 'struct']}
@@ -12,6 +13,11 @@
         cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
         cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
+        cls.w_machine = cls.space.wrap(platform.machine())
+        cls.w_resops = cls.space.newdict()
+        space = cls.space
+        for key, value in opname.items():
+            space.setitem(cls.w_resops, space.wrap(key), space.wrap(value))
 
     def test_enable(self):
         import _jitlog, struct
@@ -25,8 +31,22 @@
             assert fd.read(1) == self.mark_header
             assert fd.read(2) == self.version
             assert bool(ord(fd.read(1))) == self.is_32bit
+            strcount, = struct.unpack('<i', fd.read(4))
+            machine = fd.read(strcount)
+            assert machine == self.machine
+            # resoperations
             count, = struct.unpack('<h', fd.read(2))
+            opnames = set()
             for i in range(count):
                 opnum = struct.unpack('<h', fd.read(2))
-                strcount = struct.unpack('<i', fd.read(4))
-                fd.read(strcount)
+                strcount, = struct.unpack('<i', fd.read(4))
+                opname = fd.read(strcount)
+                opnames.append((opnum, opname))
+
+            for opnum, opname in opnames:
+                # must be known resoperation
+                assert opnum in self.resops
+                # the name must equal
+                assert self.resops[opnum] == opname
+
+
diff --git a/pypy/module/cpyext/include/pyport.h b/pypy/module/cpyext/include/pyport.h
--- a/pypy/module/cpyext/include/pyport.h
+++ b/pypy/module/cpyext/include/pyport.h
@@ -64,13 +64,6 @@
 #   error "Python needs a typedef for Py_uintptr_t in pyport.h."
 #endif /* HAVE_UINTPTR_T */
 
-/* Py_hash_t is the same size as a pointer. */
-#define SIZEOF_PY_HASH_T SIZEOF_SIZE_T
-typedef Py_ssize_t Py_hash_t;
-/* Py_uhash_t is the unsigned equivalent needed to calculate numeric hash. */
-#define SIZEOF_PY_UHASH_T SIZEOF_SIZE_T
-typedef size_t Py_uhash_t;
-
 
 /*******************************
  * stat() and fstat() fiddling *
diff --git a/pypy/module/cpyext/number.py b/pypy/module/cpyext/number.py
--- a/pypy/module/cpyext/number.py
+++ b/pypy/module/cpyext/number.py
@@ -20,16 +20,12 @@
 def PyNumber_Check(space, w_obj):
     """Returns 1 if the object o provides numeric protocols, and false otherwise.
     This function always succeeds."""
-    try:
-        space.float_w(w_obj)
+    # According to CPython, this means: w_obj is not None, and
+    # the type of w_obj has got a method __int__ or __float__.
+    if w_obj is None:
+        return 0
+    if space.lookup(w_obj, '__int__') or space.lookup(w_obj, '__float__'):
         return 1
-    except OperationError:
-        pass
-    try:
-        space.int_w(w_obj)
-        return 1
-    except OperationError:
-        pass
     return 0
 
 @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -381,6 +381,7 @@
                           ('tp_as_number.c_nb_invert', '__invert__'),
                           ('tp_as_number.c_nb_index', '__index__'),
                           ('tp_as_number.c_nb_hex', '__hex__'),
+                          ('tp_as_number.c_nb_oct', '__oct__'),
                           ('tp_str', '__str__'),
                           ('tp_repr', '__repr__'),
                           ('tp_iter', '__iter__'),
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -259,8 +259,19 @@
                     ret = PyLong_FromLong(-1);
                 Py_DECREF(obj);
                 return ret;
+             """),
+            ("has_oct", "METH_NOARGS",
+             """
+                PyObject *ret, *obj = PyLong_FromLong(42);
+                if (obj->ob_type->tp_as_number->nb_oct)
+                    ret = obj->ob_type->tp_as_number->nb_oct(obj);
+                else
+                    ret = PyLong_FromLong(-1);
+                Py_DECREF(obj);
+                return ret;
              """)])
         assert module.has_sub() == 0
         assert module.has_pow() == 0
         assert module.has_hex() == '0x2aL'
+        assert module.has_oct() == '052L'
                 
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py
--- a/pypy/module/cpyext/test/test_number.py
+++ b/pypy/module/cpyext/test/test_number.py
@@ -15,7 +15,7 @@
         assert api.PyNumber_Check(space.wraplong(-12L))
         assert api.PyNumber_Check(space.wrap(12.1))
         assert not api.PyNumber_Check(space.wrap('12'))
-        assert not api.PyNumber_Check(space.wrap(1+3j))
+        assert api.PyNumber_Check(space.wrap(1+3j))
 
     def test_number_long(self, space, api):
         w_l = api.PyNumber_Long(space.wrap(123))
@@ -151,7 +151,6 @@
              '''
                 PyObject *obj = PyTuple_GET_ITEM(args, 0);
                 int val = PyNumber_Check(obj);
-                Py_DECREF(obj);
                 return PyInt_FromLong(val);
             ''')])
         val = mod.test_PyNumber_Check(10)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -800,7 +800,7 @@
             IntLike_Type.tp_as_number = &intlike_as_number;
             intlike_as_number.nb_nonzero = intlike_nb_nonzero;
             intlike_as_number.nb_int = intlike_nb_int;
-            if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+            PyType_Ready(&IntLike_Type);
             """)
         assert not bool(module.newInt(0))
         assert bool(module.newInt(1))
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -46,6 +46,7 @@
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
+    from rpython.tool.error import offset2lineno
     bcindex = ord(bytecode.co_code[next_instr])
     opname = ""
     if 0 <= bcindex < len(opcode_method_names):
@@ -53,7 +54,8 @@
     name = bytecode.co_name
     if not name:
         name = ""
-    return (bytecode.co_filename, bytecode.co_firstlineno,
+    line = offset2lineno(bytecode, intmask(next_instr))
+    return (bytecode.co_filename, line,
             name, intmask(next_instr), opname)
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -26,8 +26,7 @@
     elif timeout == -1.0:
         microseconds = -1
     else:
-        # 0.0 => 0.0, but otherwise tends to round up
-        timeout = timeout * 1e6 + 0.999
+        timeout *= 1e6
         try:
             microseconds = ovfcheck_float_to_longlong(timeout)
         except OverflowError:
diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py
--- a/pypy/module/thread/test/test_lock.py
+++ b/pypy/module/thread/test/test_lock.py
@@ -81,7 +81,7 @@
             else:
                 got_ovf = False
                 lock.release()
-            assert (i, got_ovf) == (i, int(timeout * 1e6 + 0.999) > maxint)
+            assert (i, got_ovf) == (i, int(timeout * 1e6) > maxint)
 
     @py.test.mark.xfail(machine()=='s390x', reason='may fail under heavy load')
     def test_ping_pong(self):
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -424,29 +424,20 @@
             raise oefmt(space.w_TypeError,
                         "'%T' objects are unhashable", w_obj)
         w_result = space.get_and_call_function(w_hash, w_obj)
-        w_resulttype = space.type(w_result)
 
         # issue 2346 : returns now -2 for hashing -1 like cpython
-        if space.is_w(w_resulttype, space.w_int):
-            if space.int_w(w_result) == -1:
-                return space.wrap(-2)
-            return w_result
-        elif space.isinstance_w(w_result, space.w_int):
-            # be careful about subclasses of 'int'...
-            int_result = space.int_w(w_result)
-            if int_result == -1:
-                int_result == -2
-            return space.wrap(int_result)
+        if space.isinstance_w(w_result, space.w_int):
+            h = space.int_w(w_result)
         elif space.isinstance_w(w_result, space.w_long):
-            # be careful about subclasses of 'long'...
             bigint = space.bigint_w(w_result)
             h = bigint.hash()
-            if h == -1:
-                h = -2
-            return space.wrap(h)
         else:
             raise oefmt(space.w_TypeError,
                         "__hash__() should return an int or long")
+        # turn -1 into -2 without using a condition, which would
+        # create a potential bridge in the JIT
+        h -= (h == -1)
+        return space.wrap(h)
 
     def cmp(space, w_v, w_w):
 
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -66,17 +66,18 @@
 
     def test_wrap_various_unsigned_types(self):
         import sys
+        from rpython.rlib.rarithmetic import r_uint
         from rpython.rtyper.lltypesystem import lltype, rffi
         space = self.space
         value = sys.maxint * 2
-        x = rffi.cast(lltype.Unsigned, value)
+        x = r_uint(value)
         assert space.eq_w(space.wrap(value), space.wrap(x))
-        x = rffi.cast(rffi.UINTPTR_T, value)
+        x = rffi.cast(rffi.UINTPTR_T, r_uint(value))
         assert x > 0
         assert space.eq_w(space.wrap(value), space.wrap(x))
         value = 60000
-        x = rffi.cast(rffi.USHORT, value)
+        x = rffi.cast(rffi.USHORT, r_uint(value))
         assert space.eq_w(space.wrap(value), space.wrap(x))
         value = 200
-        x = rffi.cast(rffi.UCHAR, value)
+        x = rffi.cast(rffi.UCHAR, r_uint(value))
         assert space.eq_w(space.wrap(value), space.wrap(x))
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -128,3 +128,5 @@
     def is_w(self, obj1, obj2):
         return obj1 is obj2
 
+    def setitem(self, obj, key, value):
+        obj[key] = value
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -401,6 +401,9 @@
 class __extend__(pairtype(SomeString, SomeTuple),
                  pairtype(SomeUnicodeString, SomeTuple)):
     def mod((s_string, s_tuple)):
+        if not s_string.is_constant():
+            raise AnnotatorError("string formatting requires a constant "
+                                 "string/unicode on the left of '%'")
         is_string = isinstance(s_string, SomeString)
         is_unicode = isinstance(s_string, SomeUnicodeString)
         assert is_string or is_unicode
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4623,6 +4623,14 @@
         a = self.RPythonAnnotator()
         a.build_types(main, [int])
 
+    def test_string_mod_nonconstant(self):
+        def f(x):
+            return x % 5
+        a = self.RPythonAnnotator()
+        e = py.test.raises(AnnotatorError, a.build_types, f, [str])
+        assert ('string formatting requires a constant string/unicode'
+                in str(e.value))
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/jit/backend/x86/test/test_regloc.py b/rpython/jit/backend/x86/test/test_regloc.py
--- a/rpython/jit/backend/x86/test/test_regloc.py
+++ b/rpython/jit/backend/x86/test/test_regloc.py
@@ -147,7 +147,7 @@
             py.test.skip()
 
     def test_reuse_scratch_register(self):
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.begin_reuse_scratch_register()
         cb.MOV(ecx, heap(base_addr))
@@ -167,7 +167,7 @@
     # ------------------------------------------------------------
 
     def test_64bit_address_1(self):
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.CMP(ecx, AddressLoc(ImmedLoc(0), ImmedLoc(0), 0, base_addr))
         # this case is a CMP_rj
@@ -181,7 +181,7 @@
         assert cb.getvalue() == expected_instructions
 
     def test_64bit_address_2(self):
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(ecx, AddressLoc(ImmedLoc(0), edx, 3, base_addr))
         # this case is a CMP_ra
@@ -195,7 +195,7 @@
         assert cb.getvalue() == expected_instructions
 
     def test_64bit_address_3(self):
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(ecx, AddressLoc(edx, ImmedLoc(0), 0, base_addr))
         # this case is a CMP_rm
@@ -211,7 +211,7 @@
         assert cb.getvalue() == expected_instructions
 
     def test_64bit_address_4(self):
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.begin_reuse_scratch_register()
         assert cb._reuse_scratch_register is True
@@ -234,7 +234,7 @@
     # ------------------------------------------------------------
 
     def test_MOV_64bit_constant_into_r11(self):
-        base_constant = 0xFEDCBA9876543210
+        base_constant = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(r11, imm(base_constant))
 
@@ -245,7 +245,7 @@
         assert cb.getvalue() == expected_instructions
 
     def test_MOV_64bit_constant_into_rax(self):
-        base_constant = 0xFEDCBA9876543210
+        base_constant = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(eax, imm(base_constant))
 
@@ -256,7 +256,7 @@
         assert cb.getvalue() == expected_instructions
 
     def test_MOV_64bit_address_into_r11(self):
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(r11, heap(base_addr))
 
@@ -270,7 +270,7 @@
 
     def test_MOV_immed32_into_64bit_address_1(self):
         immed = -0x01234567
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(AddressLoc(ImmedLoc(0), ImmedLoc(0), 0, base_addr),
                ImmedLoc(immed))
@@ -286,7 +286,7 @@
 
     def test_MOV_immed32_into_64bit_address_2(self):
         immed = -0x01234567
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(AddressLoc(ImmedLoc(0), edx, 3, base_addr),
                ImmedLoc(immed))
@@ -302,7 +302,7 @@
 
     def test_MOV_immed32_into_64bit_address_3(self):
         immed = -0x01234567
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(AddressLoc(edx, ImmedLoc(0), 0, base_addr),
                ImmedLoc(immed))
@@ -320,7 +320,7 @@
 
     def test_MOV_immed32_into_64bit_address_4(self):
         immed = -0x01234567
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(AddressLoc(edx, esi, 2, base_addr), ImmedLoc(immed))
         # this case is a MOV_ai
@@ -339,7 +339,7 @@
 
     def test_MOV_immed64_into_64bit_address_1(self):
         immed = 0x0123456789ABCDEF
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(AddressLoc(ImmedLoc(0), ImmedLoc(0), 0, base_addr),
                ImmedLoc(immed))
@@ -361,7 +361,7 @@
 
     def test_MOV_immed64_into_64bit_address_2(self):
         immed = 0x0123456789ABCDEF
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(AddressLoc(ImmedLoc(0), edx, 3, base_addr),
                ImmedLoc(immed))
@@ -383,7 +383,7 @@
 
     def test_MOV_immed64_into_64bit_address_3(self):
         immed = 0x0123456789ABCDEF
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(AddressLoc(eax, ImmedLoc(0), 0, base_addr),
                ImmedLoc(immed))
@@ -407,7 +407,7 @@
 
     def test_MOV_immed64_into_64bit_address_4(self):
         immed = 0x0123456789ABCDEF
-        base_addr = 0xFEDCBA9876543210
+        base_addr = intmask(0xFEDCBA9876543210)
         cb = LocationCodeBuilder64()
         cb.MOV(AddressLoc(edx, eax, 2, base_addr), ImmedLoc(immed))
         # this case is a MOV_ai
diff --git a/rpython/jit/backend/x86/test/test_ztranslation_call_assembler.py b/rpython/jit/backend/x86/test/test_ztranslation_call_assembler.py
--- a/rpython/jit/backend/x86/test/test_ztranslation_call_assembler.py
+++ b/rpython/jit/backend/x86/test/test_ztranslation_call_assembler.py
@@ -4,6 +4,16 @@
 from rpython.jit.backend.x86.arch import WORD
 import sys
 
+
+# On Windows, this test crashes obscurely, but only if compiled with
+# Boehm, not if run with no GC at all.  So for now we'll assume it is
+# really a Boehm bug, or maybe a Boehm-on-Windows-specific issue, and
+# skip.
+if sys.platform == 'win32':
+    import py
+    py.test.skip("crashes on Windows (Boehm issue?)")
+
+
 class TestTranslationCallAssemblerX86(TranslationTestCallAssembler):
     def _check_cbuilder(self, cbuilder):
         #We assume here that we have sse2.  If not, the CPUClass
diff --git a/rpython/jit/backend/zarch/test/test_assembler.py b/rpython/jit/backend/zarch/test/test_assembler.py
--- a/rpython/jit/backend/zarch/test/test_assembler.py
+++ b/rpython/jit/backend/zarch/test/test_assembler.py
@@ -21,6 +21,7 @@
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.longlong2float import (float2longlong,
         DOUBLE_ARRAY_PTR, singlefloat2uint_emulator)
+from rpython.rlib.rarithmetic import r_uint, intmask
 import ctypes
 
 CPU = getcpuclass()
@@ -168,7 +169,7 @@
     def test_load_byte_zero_extend(self):
         adr = self.a.datablockwrapper.malloc_aligned(16, 16)
         data = rffi.cast(rffi.CArrayPtr(rffi.ULONG), adr)
-        data[0] = rffi.cast(rffi.ULONG,0xffffFFFFffffFF02)
+        data[0] = rffi.cast(rffi.ULONG, intmask(0xffffFFFFffffFF02))
         self.a.mc.load_imm(r.r3, adr+7)
         self.a.mc.LLGC(r.r2, loc.addr(0,r.r3))
         self.a.mc.BCR(con.ANY, r.r14)
@@ -177,7 +178,7 @@
     def test_load_byte_and_imm(self):
         adr = self.a.datablockwrapper.malloc_aligned(16, 16)
         data = rffi.cast(rffi.CArrayPtr(rffi.ULONG), adr)
-        data[0] = rffi.cast(rffi.ULONG,0xffffFFFFffff0001)
+        data[0] = rffi.cast(rffi.ULONG, intmask(0xffffFFFFffff0001))
         self.a.mc.load_imm(r.r3, adr)
         self.a.mc.LG(r.r2, loc.addr(0,r.r3))
         self.a.mc.LLGC(r.r2, loc.addr(7,r.r3))
diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -11,7 +11,7 @@
 from rpython.rlib.jit_libffi import (types, CIF_DESCRIPTION, FFI_TYPE_PP,
                                      jit_ffi_call)
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.rarithmetic import intmask, r_longlong, r_singlefloat
+from rpython.rlib.rarithmetic import intmask, r_longlong, r_singlefloat, r_uint
 from rpython.rlib.longlong2float import float2longlong
 
 def get_description(atypes, rtype):
@@ -230,8 +230,8 @@
 
     def test_handle_unsigned(self):
         self._run([types.ulong], types.ulong,
-                  [rffi.cast(rffi.ULONG, sys.maxint + 91348)],
-                  rffi.cast(rffi.ULONG, sys.maxint + 4242))
+                  [rffi.cast(rffi.ULONG, r_uint(sys.maxint + 91348))],
+                  rffi.cast(rffi.ULONG, r_uint(sys.maxint + 4242)))
 
     def test_handle_unsignedchar(self):
         self._run([types.uint8], types.uint8,
diff --git a/rpython/jit/metainterp/test/test_memmgr.py b/rpython/jit/metainterp/test/test_memmgr.py
--- a/rpython/jit/metainterp/test/test_memmgr.py
+++ b/rpython/jit/metainterp/test/test_memmgr.py
@@ -248,8 +248,8 @@
         tokens = [t() for t in get_stats().jitcell_token_wrefs]
         # Some loops have been freed
         assert None in tokens
-        # Loop with number 0, h(), has not been freed
-        assert 0 in [t.number for t in tokens if t]
+        # Loop with number 1, h(), has not been freed
+        assert 1 in [t.number for t in tokens if t]
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/tl/tla/targettla.py b/rpython/jit/tl/tla/targettla.py
--- a/rpython/jit/tl/tla/targettla.py
+++ b/rpython/jit/tl/tla/targettla.py
@@ -4,9 +4,16 @@
 
 
 def entry_point(args):
-    """Main entry point of the stand-alone executable:
-    takes a list of strings and returns the exit code.
-    """
+    for i in range(len(argv)):
+        if argv[i] == "--jit":
+            if len(argv) == i + 1:
+                print "missing argument after --jit"
+                return 2
+            jitarg = argv[i + 1]
+            del argv[i:i+2]
+            jit.set_user_param(jitdriver, jitarg)
+            break
+
     if len(args) < 3:
         print "Usage: %s filename x" % (args[0],)
         return 2
@@ -26,7 +33,7 @@
     return bytecode
 
 def target(driver, args):
-    return entry_point, None
+    return entry_point
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/tl/tla/tla.py b/rpython/jit/tl/tla/tla.py
--- a/rpython/jit/tl/tla/tla.py
+++ b/rpython/jit/tl/tla/tla.py
@@ -60,19 +60,34 @@
 
 # ____________________________________________________________
 
-CONST_INT = 1
-POP       = 2
-ADD       = 3
-RETURN    = 4
-JUMP_IF   = 5
-DUP       = 6
-SUB       = 7
-NEWSTR    = 8
+OPNAMES = []
+HASARG = []
+
+def define_op(name, has_arg=False):
+    globals()[name] = len(OPNAMES)
+    OPNAMES.append(name)
+    HASARG.append(has_arg)
+
+define_op("CONST_INT", True)
+define_op("POP")
+define_op("ADD")
+define_op("RETURN")
+define_op("JUMP_IF", True)
+define_op("DUP")
+define_op("SUB")
+define_op("NEWSTR", True)
+
 
 # ____________________________________________________________
 
 def get_printable_location(pc, bytecode):
-    return str(pc)
+    op = ord(bytecode[pc])
+    name = OPNAMES[op]
+    if HASARG[op]:
+        arg = str(ord(bytecode[pc + 1]))
+    else:
+        arg = ''
+    return "%s: %s %s" % (pc, name, arg)
 
 jitdriver = JitDriver(greens=['pc', 'bytecode'],
                       reds=['self'],
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -3,6 +3,7 @@
 import weakref
 import struct
 import os
+import platform
 from rpython.rlib import jit
 from rpython.tool.udir import udir
 from rpython.tool.version import rpythonroot
@@ -282,14 +283,16 @@
 
 IS_32_BIT = sys.maxint == 2**31-1
 
+MACHINE_NAME = platform.machine()
+
 def assemble_header():
     version = JITLOG_VERSION_16BIT_LE
     count = len(resoperations.opname)
     is_32bit = chr(0x1)
     if not IS_32_BIT:
         is_32bit = chr(0x0)
-    content = [version, is_32bit, MARK_RESOP_META,
-               encode_le_16bit(count)]
+    content = [version, is_32bit, encode_str(MACHINE_NAME),
+               MARK_RESOP_META, encode_le_16bit(count)]
     for opnum, opname in resoperations.opname.items():
         content.append(encode_le_16bit(opnum))
         content.append(encode_str(opname.lower()))
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -404,6 +404,8 @@
 def test_int_c_div_mod(x, y):
     assert int_c_div(~x, y) == -(abs(~x) // y)
     assert int_c_div( x,-y) == -(x // y)
+    if (x, y) == (sys.maxint, 1):
+        py.test.skip("would overflow")
     assert int_c_div(~x,-y) == +(abs(~x) // y)
     for x1 in [x, ~x]:
         for y1 in [y, -y]:
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -812,8 +812,10 @@
         if tp is long:
             if -maxint-1 <= val <= maxint:
                 return Signed
+            elif longlongmask(val) == val:
+                return SignedLongLong
             else:
-                return SignedLongLong
+                raise OverflowError("integer %r is out of bounds" % (val,))
         if tp is bool:
             return Bool
         if issubclass(tp, base_int):
diff --git a/rpython/rtyper/rrange.py b/rpython/rtyper/rrange.py
--- a/rpython/rtyper/rrange.py
+++ b/rpython/rtyper/rrange.py
@@ -199,8 +199,11 @@
         self.r_baseiter = r_baseiter
         self.lowleveltype = r_baseiter.lowleveltype
         # only supports for now enumerate() on sequence types whose iterators
-        # have a method ll_getnextindex.  It's easy to add one for most
-        # iterator types, but I didn't do it so far.
+        # have a method ll_getnextindex.  It could be added for most
+        # iterator types, but it's a bit messy for no clear benefit.
+        if not hasattr(r_baseiter, 'll_getnextindex'):
+            raise TyperError("not implemented for now: enumerate(x) where x "
+                             "is not a regular list (got %r)" % (r_baseiter,))
         self.ll_getnextindex = r_baseiter.ll_getnextindex
 
     def rtype_next(self, hop):
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -591,7 +591,9 @@
 
 class __extend__(pairtype(IntegerRepr, AbstractStringRepr)):
     def rtype_mul((r_int, r_str), hop):
-        return pair(r_str, r_int).rtype_mul(hop)
+        str_repr = r_str.repr
+        v_int, v_str = hop.inputargs(Signed, str_repr)
+        return hop.gendirectcall(r_str.ll.ll_str_mul, v_str, v_int)
     rtype_inplace_mul = rtype_mul
 
 
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -220,11 +220,12 @@
         const = self.const
         def fn(i, mul):
             s = ["", "a", "aba"][i]
-            return s * mul
+            return s * mul + mul * s
         for i in xrange(3):
             for m in [0, 1, 4]:
+                res1 = fn(i, m)
                 res = self.interpret(fn, [i, m])
-                assert self.ll_to_string(res) == fn(i, m)
+                assert self.ll_to_string(res) == res1
 
     def test_is_none(self):
         const = self.const
diff --git a/rpython/tool/error.py b/rpython/tool/error.py
--- a/rpython/tool/error.py
+++ b/rpython/tool/error.py
@@ -158,6 +158,8 @@
 
 @jit.elidable
 def offset2lineno(c, stopat):
+    # even position in lnotab denote byte increments, odd line increments.
+    # see dis.findlinestarts in the python std. library for more details
     tab = c.co_lnotab
     line = c.co_firstlineno
     addr = 0
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -422,14 +422,11 @@
             mk.definition('PROFOPT', profopt)
 
         rules = [
-            ('clean', '', 'rm -f $(OBJECTS) $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES) *.gc?? ../module_cache/*.gc??'),
-            ('clean_noprof', '', 'rm -f $(OBJECTS) $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES)'),
             ('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" debug_target'),
             ('debug_exc', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DDO_LOG_EXC" debug_target'),
             ('debug_mem', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DPYPY_USE_TRIVIAL_MALLOC" debug_target'),
             ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(DEFAULT_TARGET)'),
             ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
-            ('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DMAX_STACK_SIZE=8192000 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
             ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(DEFAULT_TARGET)'),
             ]
         if self.has_profopt():
@@ -443,6 +440,17 @@
         for rule in rules:
             mk.rule(*rule)
 
+        if self.translator.platform.name == 'msvc':
+            mk.rule('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -Od -DMAX_STACK_SIZE=8192000 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
+            wildcards = '..\*.obj ..\*.pdb ..\*.lib ..\*.dll ..\*.manifest ..\*.exp *.pch'
+            cmd =  r'del /s %s $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES)' % wildcards
+            mk.rule('clean', '',  cmd + ' *.gc?? ..\module_cache\*.gc??')
+            mk.rule('clean_noprof', '', cmd)
+        else:
+            mk.rule('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DMAX_STACK_SIZE=8192000 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
+            mk.rule('clean', '', 'rm -f $(OBJECTS) $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES) *.gc?? ../module_cache/*.gc??')
+            mk.rule('clean_noprof', '', 'rm -f $(OBJECTS) $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES)')
+
         #XXX: this conditional part is not tested at all
         if self.config.translation.gcrootfinder == 'asmgcc':
             if self.translator.platform.name == 'msvc':
@@ -507,7 +515,7 @@
                 else:
                     mk.definition('DEBUGFLAGS', '-O1 -g')
         if self.translator.platform.name == 'msvc':
-            mk.rule('debug_target', '$(DEFAULT_TARGET)', 'rem')
+            mk.rule('debug_target', '$(DEFAULT_TARGET) $(WTARGET)', 'rem')
         else:
             mk.rule('debug_target', '$(DEFAULT_TARGET)', '#')
         mk.write()

From pypy.commits at gmail.com  Wed Aug  3 07:34:45 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 03 Aug 2016 04:34:45 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merged default,
 added message to exception thrown in vecopt
Message-ID: <57a1d6d5.c3cb1c0a.fb5be.a50f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86001:b6da37166dc2
Date: 2016-08-03 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/b6da37166dc2/

Log:	merged default, added message to exception thrown in vecopt

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -109,3 +109,13 @@
 .. branch: jitlog-exact-source-lines
 
 Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -157,11 +157,13 @@
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
         try:
+            keepalives = [None] * len(args_w)    # None or strings
             for i in range(len(args_w)):
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                 w_obj = args_w[i]
                 argtype = self.fargs[i]
-                if argtype.convert_argument_from_object(data, w_obj):
+                if argtype.convert_argument_from_object(data, w_obj,
+                                                        keepalives, i):
                     # argtype is a pointer type, and w_obj a list/tuple/str
                     mustfree_max_plus_1 = i + 1
 
@@ -177,9 +179,13 @@
                 if isinstance(argtype, W_CTypePointer):
                     data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                     flag = get_mustfree_flag(data)
+                    raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                     if flag == 1:
-                        raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
+                    elif flag >= 4:
+                        value = keepalives[i]
+                        assert value is not None
+                        rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
             lltype.free(buffer, flavor='raw')
             keepalive_until_here(args_w)
         return w_res
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -83,7 +83,7 @@
         raise oefmt(space.w_TypeError, "cannot initialize cdata '%s'",
                     self.name)
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         self.convert_from_object(cdata, w_ob)
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,8 +14,8 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
+    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
     length = -1
 
     def __init__(self, space, size, extra, extra_position, ctitem,
@@ -28,6 +28,9 @@
         #  - for functions, it is the return type
         self.ctitem = ctitem
         self.can_cast_anything = could_cast_anything and ctitem.cast_anything
+        self.accept_str = (self.can_cast_anything or
+                            (ctitem.is_primitive_integer and
+                             ctitem.size == rffi.sizeof(lltype.Char)))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -70,9 +73,7 @@
                 pass
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
-        elif (self.can_cast_anything or
-              (self.ctitem.is_primitive_integer and
-               self.ctitem.size == rffi.sizeof(lltype.Char))):
+        elif self.accept_str:
             if not space.isinstance_w(w_ob, space.w_str):
                 raise self._convert_error("str or list or tuple", w_ob)
             s = space.str_w(w_ob)
@@ -260,8 +261,16 @@
         else:
             return lltype.nullptr(rffi.CCHARP.TO)
 
-    def _prepare_pointer_call_argument(self, w_init, cdata):
+    def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
         space = self.space
+        if self.accept_str and space.isinstance_w(w_init, space.w_str):
+            # special case to optimize strings passed to a "char *" argument
+            value = w_init.str_w(space)
+            keepalives[i] = value
+            buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
+            rffi.cast(rffi.CCHARPP, cdata)[0] = buf
+            return ord(buf_flag)    # 4, 5 or 6
+        #
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
@@ -297,10 +306,11 @@
         rffi.cast(rffi.CCHARPP, cdata)[0] = result
         return 1
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
         result = (not isinstance(w_ob, cdataobj.W_CData) and
-                  self._prepare_pointer_call_argument(w_ob, cdata))
+                  self._prepare_pointer_call_argument(w_ob, cdata,
+                                                      keepalives, i))
         if result == 0:
             self.convert_from_object(cdata, w_ob)
         set_mustfree_flag(cdata, result)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -353,7 +353,7 @@
 'array.array' or numpy arrays."""
         #
         w_ctchara = newtype._new_chara_type(self.space)
-        return func.from_buffer(self.space, w_ctchara, w_python_buffer)
+        return func._from_buffer(self.space, w_ctchara, w_python_buffer)
 
 
     @unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated
+from rpython.rlib import jit
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -132,17 +133,66 @@
         raise oefmt(space.w_TypeError,
                     "needs 'char[]', got '%s'", w_ctype.name)
     #
+    return _from_buffer(space, w_ctype, w_x)
+
+def _from_buffer(space, w_ctype, w_x):
     buf = _fetch_as_read_buffer(space, w_x)
-    try:
-        _cdata = buf.get_raw_address()
-    except ValueError:
-        raise oefmt(space.w_TypeError,
-                    "from_buffer() got a '%T' object, which supports the "
-                    "buffer interface but cannot be rendered as a plain "
-                    "raw address on PyPy", w_x)
+    if space.isinstance_w(w_x, space.w_str):
+        _cdata = get_raw_address_of_string(space, w_x)
+    else:
+        try:
+            _cdata = buf.get_raw_address()
+        except ValueError:
+            raise oefmt(space.w_TypeError,
+                        "from_buffer() got a '%T' object, which supports the "
+                        "buffer interface but cannot be rendered as a plain "
+                        "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
 
+# ____________________________________________________________
+
+class RawBytes(object):
+    def __init__(self, string):
+        self.ptr = rffi.str2charp(string, track_allocation=False)
+    def __del__(self):
+        rffi.free_charp(self.ptr, track_allocation=False)
+
+class RawBytesCache(object):
+    def __init__(self, space):
+        from pypy.interpreter.baseobjspace import W_Root
+        from rpython.rlib import rweakref
+        self.wdict = rweakref.RWeakKeyDictionary(W_Root, RawBytes)
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(space, w_x):
+    """Special case for ffi.from_buffer(string).  Returns a 'char *' that
+    is valid as long as the string object is alive.  Two calls to
+    ffi.from_buffer(same_string) are guaranteed to return the same pointer.
+    """
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rtyper.lltypesystem.rstr import STR
+    from rpython.rtyper.lltypesystem import llmemory
+    from rpython.rlib import rgc
+
+    cache = space.fromcache(RawBytesCache)
+    rawbytes = cache.wdict.get(w_x)
+    if rawbytes is None:
+        data = space.str_w(w_x)
+        if we_are_translated() and not rgc.can_move(data):
+            lldata = llstr(data)
+            data_start = (llmemory.cast_ptr_to_adr(lldata) +
+                          rffi.offsetof(STR, 'chars') +
+                          llmemory.itemoffsetof(STR.chars, 0))
+            data_start = rffi.cast(rffi.CCHARP, data_start)
+            data_start[len(data)] = '\x00'   # write the final extra null
+            return data_start
+        rawbytes = RawBytes(data)
+        cache.wdict.set(w_x, rawbytes)
+    return rawbytes.ptr
+
+# ____________________________________________________________
+
 
 def unsafe_escaping_ptr_for_ptr_or_array(w_cdata):
     if not w_cdata.ctype.is_nonfunc_pointer_or_array:
diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py
--- a/pypy/module/_cffi_backend/parse_c_type.py
+++ b/pypy/module/_cffi_backend/parse_c_type.py
@@ -97,11 +97,8 @@
                                   [rffi.INT], rffi.CCHARP)
 
 def parse_c_type(info, input):
-    p_input = rffi.str2charp(input)
-    try:
+    with rffi.scoped_view_charp(input) as p_input:
         res = ll_parse_c_type(info, p_input)
-    finally:
-        rffi.free_charp(p_input)
     return rffi.cast(lltype.Signed, res)
 
 NULL_CTX = lltype.nullptr(PCTX.TO)
@@ -130,15 +127,13 @@
     return rffi.getintfield(src_ctx, 'c_num_types')
 
 def search_in_globals(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_globals(ctx, c_name,
-                                  rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_globals(ctx, c_name,
+                                      rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
 
 def search_in_struct_unions(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_struct_unions(ctx, c_name,
-                                        rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_struct_unions(ctx, c_name,
+                                            rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3330,13 +3330,18 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharA = new_array_type(BCharP, None)
-    py.test.raises(TypeError, from_buffer, BCharA, b"foo")
+    p1 = from_buffer(BCharA, b"foo")
+    assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # from_buffer(buffer(b"foo")) does not work, because it's not
+        # implemented on pypy; only from_buffer(b"foo") works.
         py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
         py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
     try:
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -401,21 +401,20 @@
             _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
         from rpython.rlib import rwin32
 
-        charp = rffi.str2charp(buf)
-        written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                    flavor='raw')
-        try:
-            result = _WriteFile(
-                self.handle, rffi.ptradd(charp, offset),
-                size, written_ptr, rffi.NULL)
+        with rffi.scoped_view_charp(buf) as charp:
+            written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
+                                        flavor='raw')
+            try:
+                result = _WriteFile(
+                    self.handle, rffi.ptradd(charp, offset),
+                    size, written_ptr, rffi.NULL)
 
-            if (result == 0 and
-                rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
-                raise oefmt(space.w_ValueError,
-                            "Cannot send %d bytes over connection", size)
-        finally:
-            rffi.free_charp(charp)
-            lltype.free(written_ptr, flavor='raw')
+                if (result == 0 and
+                    rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
+                    raise oefmt(space.w_ValueError,
+                                "Cannot send %d bytes over connection", size)
+            finally:
+                lltype.free(written_ptr, flavor='raw')
 
     def do_recv_string(self, space, buflength, maxlength):
         from pypy.module._multiprocessing.interp_win32 import (
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -20,7 +20,8 @@
     def _getfunc(space, CDLL, w_name, w_argtypes, w_restype):
         argtypes_w, argtypes, w_restype, restype = unpack_argtypes(
             space, w_argtypes, w_restype)
-        if space.isinstance_w(w_name, space.w_str):
+        if (space.isinstance_w(w_name, space.w_str) or
+                space.isinstance_w(w_name, space.w_unicode)):
             name = space.str_w(w_name)
             try:
                 func = CDLL.cdll.getpointer(name, argtypes, restype,
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -135,7 +135,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
@@ -147,7 +147,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
@@ -181,7 +181,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
@@ -193,7 +193,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
@@ -228,7 +228,7 @@
 
         Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
         bound on the entropy contained in string."""
-        with rffi.scoped_str2charp(string) as buf:
+        with rffi.scoped_nonmovingbuffer(string) as buf:
             libssl_RAND_add(buf, len(string), entropy)
 
     def RAND_status(space):
diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -537,9 +537,8 @@
     releasegil=ts_helper,
     compilation_info=backend.eci)
 def c_charp2stdstring(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2stdstring(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2stdstring(charp)
     return result
 _c_stdstring2stdstring = rffi.llexternal(
     "cppyy_stdstring2stdstring",
diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -82,9 +82,8 @@
     releasegil=ts_helper,
     compilation_info=eci)
 def c_charp2TString(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2TString(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2TString(charp)
     return result
 _c_TString2TString = rffi.llexternal(
     "cppyy_TString2TString",
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -65,6 +65,7 @@
                 else:    # only other use is sring
                     n = len(obj._string)
                     assert raw_string == rffi.cast(rffi.CCHARP, 0)
+                    # XXX could use rffi.get_nonmovingbuffer_final_null()
                     raw_string = rffi.str2charp(obj._string)
                     data = rffi.cast(rffi.CCHARPP, data)
                     data[0] = raw_string
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -96,7 +96,8 @@
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
             py_str.c_ob_size, len(s))
-    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    with rffi.scoped_nonmovingbuffer(s) as s_ptr:
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
     py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -133,6 +133,12 @@
         # You cannot assing character format codes as restype any longer
         raises(TypeError, setattr, f, "restype", "i")
 
+    def test_unicode_function_name(self):
+        f = dll[u'_testfunc_i_bhilfd']
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_int
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        assert result == 21
 
     def test_truncate_python_longs(self):
         f = dll._testfunc_i_bhilfd
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-# hypothesis is used for test generation on untranslated jit tests
+# hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -883,6 +883,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                               self.cpu.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -280,7 +280,7 @@
     concrete_type = '\x00'
 
     def __init__(self, basesize, itemsize, lendescr, flag, is_pure=False, concrete_type='\x00'):
-        self.basesize = basesize
+        self.basesize = basesize    # this includes +1 for STR
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
@@ -676,7 +676,7 @@
 
 def unpack_arraydescr(arraydescr):
     assert isinstance(arraydescr, ArrayDescr)
-    ofs = arraydescr.basesize
+    ofs = arraydescr.basesize    # this includes +1 for STR
     size = arraydescr.itemsize
     sign = arraydescr.is_item_signed()
     return size, ofs, sign
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -265,6 +265,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_load_or_indexed(op, op.getarg(0), op.getarg(1),
                                          itemsize, itemsize, basesize, NOT_SIGNED)
         elif opnum == rop.UNICODEGETITEM:
@@ -276,6 +277,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_store_or_indexed(op, op.getarg(0), op.getarg(1), op.getarg(2),
                                          itemsize, itemsize, basesize)
         elif opnum == rop.UNICODESETITEM:
diff --git a/rpython/jit/backend/llsupport/symbolic.py b/rpython/jit/backend/llsupport/symbolic.py
--- a/rpython/jit/backend/llsupport/symbolic.py
+++ b/rpython/jit/backend/llsupport/symbolic.py
@@ -29,7 +29,7 @@
 def get_array_token(T, translate_support_code):
     # T can be an array or a var-sized structure
     if translate_support_code:
-        basesize = llmemory.sizeof(T, 0)
+        basesize = llmemory.sizeof(T, 0)     # this includes +1 for STR
         if isinstance(T, lltype.Struct):
             SUBARRAY = getattr(T, T._arrayfld)
             itemsize = llmemory.sizeof(SUBARRAY.OF)
@@ -57,6 +57,7 @@
             assert carray.length.size == WORD
             ofs_length = before_array_part + carray.length.offset
         basesize = before_array_part + carray.items.offset
+        basesize += T._hints.get('extra_item_after_alloc', 0)  # +1 for STR
         carrayitem = ll2ctypes.get_ctypes_type(T.OF)
         itemsize = ctypes.sizeof(carrayitem)
     return basesize, itemsize, ofs_length
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -435,8 +435,10 @@
 def test_bytearray_descr():
     c0 = GcCache(False)
     descr = get_array_descr(c0, rstr.STR)   # for bytearray
+    # note that we get a basesize that has 1 extra byte for the final null char
+    # (only for STR)
     assert descr.flag == FLAG_UNSIGNED
-    assert descr.basesize == struct.calcsize("PP")         # hash, length
+    assert descr.basesize == struct.calcsize("PP") + 1     # hash, length, extra
     assert descr.lendescr.offset == struct.calcsize("P")   # hash
     assert not descr.is_array_of_pointers()
 
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -647,6 +647,9 @@
         """)
 
     def test_rewrite_assembler_newstr_newunicode(self):
+        # note: strdescr.basesize already contains the extra final character,
+        # so that's why newstr(14) is rounded up to 'basesize+15' and not
+        # 'basesize+16'.
         self.check_rewrite("""
             [i2]
             p0 = newstr(14)
@@ -657,12 +660,12 @@
         """, """
             [i2]
             p0 = call_malloc_nursery(                                \
-                      %(strdescr.basesize + 16 * strdescr.itemsize + \
+                      %(strdescr.basesize + 15 * strdescr.itemsize + \
                         unicodedescr.basesize + 10 * unicodedescr.itemsize)d)
             gc_store(p0, 0,  %(strdescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p0, %(strlendescr.offset)s, 14, %(strlendescr.field_size)s)
             gc_store(p0, 0,  0, %(strhashdescr.field_size)s)
-            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
+            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 15 * strdescr.itemsize)d)
             gc_store(p1, 0,  %(unicodedescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p1, %(unicodelendescr.offset)s, 10, %(unicodelendescr.field_size)s)
             gc_store(p1, 0,  0, %(unicodehashdescr.field_size)s)
@@ -1240,14 +1243,14 @@
         #               'i3 = gc_load_i(p0,i5,%(unicodedescr.itemsize)d)'],
         [True,  (4,),  'i3 = strgetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,1,'
-                       '%(strdescr.basesize)d,1)'],
+                       '%(strdescr.basesize-1)d,1)'],
         #[False, (4,),  'i3 = strgetitem(p0,i1)' '->'
-        #               'i5 = int_add(i1, %(strdescr.basesize)d);'
+        #               'i5 = int_add(i1, %(strdescr.basesize-1)d);'
         #               'i3 = gc_load_i(p0,i5,1)'],
         ## setitem str/unicode
         [True, (4,),  'i3 = strsetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,1,'
-                               '%(strdescr.basesize)d,1)'],
+                               '%(strdescr.basesize-1)d,1)'],
         [True, (2,4), 'i3 = unicodesetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,'
                                  '%(unicodedescr.itemsize)d,'
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -3,7 +3,7 @@
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
 from rpython.rlib.jit import promote, _get_virtualizable_token
-from rpython.rlib import jit_hooks, rposix
+from rpython.rlib import jit_hooks, rposix, rgc
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
 from rpython.jit.backend.detect_cpu import getcpuclass
@@ -11,7 +11,7 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 from rpython.config.config import ConfigError
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.rlib.rjitlog import rjitlog as jl
 
 
@@ -29,6 +29,7 @@
         # - floats neg and abs
         # - cast_int_to_float
         # - llexternal with macro=True
+        # - extra place for the zero after STR instances
 
         class BasicFrame(object):
             _virtualizable_ = ['i']
@@ -56,7 +57,7 @@
             return ("/home.py",0,0)
 
         jitdriver = JitDriver(greens = [],
-                              reds = ['total', 'frame', 'j'],
+                              reds = ['total', 'frame', 'prev_s', 'j'],
                               virtualizables = ['frame'],
                               get_location = get_location)
         def f(i, j):
@@ -68,9 +69,12 @@
             total = 0
             frame = Frame(i)
             j = float(j)
+            prev_s = rstr.mallocstr(16)
             while frame.i > 3:
-                jitdriver.can_enter_jit(frame=frame, total=total, j=j)
-                jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                jitdriver.can_enter_jit(frame=frame, total=total, j=j,
+                                        prev_s=prev_s)
+                jitdriver.jit_merge_point(frame=frame, total=total, j=j,
+                                          prev_s=prev_s)
                 _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
@@ -82,6 +86,11 @@
                 k = myabs1(myabs2(j))
                 if k - abs(j):  raise ValueError
                 if k - abs(-j): raise ValueError
+                s = rstr.mallocstr(16)
+                rgc.ll_write_final_null_char(s)
+                rgc.ll_write_final_null_char(prev_s)
+                if (frame.i & 3) == 0:
+                    prev_s = s
             return chr(total % 253)
         #
         class Virt2(object):
diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py
--- a/rpython/jit/backend/llsupport/vector_ext.py
+++ b/rpython/jit/backend/llsupport/vector_ext.py
@@ -192,9 +192,11 @@
                 continue
             curvecinfo = forwarded_vecinfo(arg)
             if curvecinfo.bytesize != bytesize:
-                raise NotAVectorizeableLoop()
+                raise NotAVectorizeableLoop("op match size first type failed %d != %d" % \
+                        (curvecinfo.bytesize != bytesize))
             if curvecinfo.datatype != datatype:
-                raise NotAVectorizeableLoop()
+                raise NotAVectorizeableLoop("op match size first type failed (datatype). %s != %s" % \
+                        (curvecinfo.datatype != datatype))
         return None
 
 TR_ANY = TypeRestrict()
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -995,6 +995,7 @@
             basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                         self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             scale = 0
 
         self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1685,25 +1685,6 @@
         dest_addr = AddressLoc(base_loc, ofs_loc, scale, offset_loc.value)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
-    def genop_discard_strsetitem(self, op, arglocs):
-        base_loc, ofs_loc, val_loc = arglocs
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                              self.cpu.translate_support_code)
-        assert itemsize == 1
-        dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize)
-        self.mc.MOV8(dest_addr, val_loc.lowest8bits())
-
-    def genop_discard_unicodesetitem(self, op, arglocs):
-        base_loc, ofs_loc, val_loc = arglocs
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                              self.cpu.translate_support_code)
-        if itemsize == 4:
-            self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc)
-        elif itemsize == 2:
-            self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc)
-        else:
-            assert 0, itemsize
-
     # genop_discard_setfield_raw = genop_discard_setfield_gc
 
     def genop_math_read_timestamp(self, op, arglocs, resloc):
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1219,6 +1219,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                                   self.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self.assembler.load_effective_addr(ofsloc, ofs_items, scale,
                                            resloc, baseloc)
diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py
--- a/rpython/jit/backend/zarch/opassembler.py
+++ b/rpython/jit/backend/zarch/opassembler.py
@@ -991,6 +991,7 @@
             basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                         self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             scale = 0
 
         # src and src_len are tmp registers
diff --git a/rpython/jit/metainterp/jitexc.py b/rpython/jit/metainterp/jitexc.py
--- a/rpython/jit/metainterp/jitexc.py
+++ b/rpython/jit/metainterp/jitexc.py
@@ -62,8 +62,10 @@
             self.red_int, self.red_ref, self.red_float)
 
 class NotAVectorizeableLoop(JitException):
+    def __init__(self, msg="<empty>"):
+        self.msg = msg
     def __str__(self):
-        return 'NotAVectorizeableLoop()'
+        return 'NotAVectorizeableLoop(%s)' % self.msg
 
 class NotAProfitableLoop(JitException):
     def __str__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -146,12 +146,14 @@
         #
         info.label_op = loop.label
         return info, loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
-    except NotAVectorizeableLoop:
+    except NotAVectorizeableLoop as e:
         debug_stop("vec-opt-loop")
+        debug_print("failed to vectorize loop. reason: %s" % str(e))
         # vectorization is not possible
         return loop_info, version.loop.finaloplist()
     except NotAProfitableLoop:
         debug_stop("vec-opt-loop")
+        debug_print("failed to vectorize loop, cost model indicated it is not profitable")
         # cost model says to skip this loop
         return loop_info, version.loop.finaloplist()
     except Exception as e:
@@ -231,7 +233,8 @@
         if vsize == 0 or byte_count == 0 or loop.label.getopnum() != rop.LABEL:
             # stop, there is no chance to vectorize this trace
             # we cannot optimize normal traces (if there is no label)
-            raise NotAVectorizeableLoop()
+            raise NotAVectorizeableLoop("vsize %d byte_count %d not label? %d" % \
+                    (vsize, byte_count, loop.label.getopnum() != rop.LABEL))
 
         # find index guards and move to the earliest position
         graph = self.analyse_index_calculations(loop)
@@ -253,7 +256,7 @@
         state = VecScheduleState(graph, self.packset, self.cpu, costmodel)
         self.schedule(state)
         if not state.profitable():
-            raise NotAProfitableLoop()
+            raise NotAProfitableLoop
         return graph.index_vars
 
     def unroll_loop_iterations(self, loop, unroll_count):
@@ -429,7 +432,7 @@
             intersecting edges.
         """
         if len(self.packset.packs) == 0:
-            raise NotAVectorizeableLoop()
+            raise NotAVectorizeableLoop("packset is empty")
         i = 0
         j = 0
         end_ij = len(self.packset.packs)
@@ -661,7 +664,7 @@
                 if forward and origin_pack.is_accumulating():
                     # in this case the splitted accumulator must
                     # be combined. This case is not supported
-                    raise NotAVectorizeableLoop()
+                    raise NotAVectorizeableLoop("splitted accum must be flushed here (not supported)")
                 #
                 if self.contains_pair(lnode, rnode):
                     return None
diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -1381,7 +1381,7 @@
             return result
 
         def indirection(arg):
-            return interp(arg)
+            return interp(arg) + 1
 
         def run_interp(n):
             f = hint(Frame(n), access_directly=True)
diff --git a/rpython/memory/gcheader.py b/rpython/memory/gcheader.py
--- a/rpython/memory/gcheader.py
+++ b/rpython/memory/gcheader.py
@@ -11,7 +11,21 @@
     def __init__(self, HDR):
         """NOT_RPYTHON"""
         self.HDR = HDR
-        self.obj2header = weakref.WeakKeyDictionary()
+        #
+        # The following used to be a weakref.WeakKeyDictionary(), but
+        # the problem is that if you have a gcobj which has already a
+        # weakref cached on it and the hash already cached in that
+        # weakref, and later the hash of the gcobj changes (because it
+        # is ll2ctypes-ified), then that gcobj cannot be used as a key
+        # in a WeakKeyDictionary any more: from this point on,
+        # 'ref(gcobj)' and 'ref(gcobj, callback)' return two objects
+        # with different hashes... and so e.g. the sequence of
+        # operations 'obj2header[x]=y; assert x in obj2header' fails.
+        #
+        # Instead, just use a regular dictionary and hope that not too
+        # many objects would be reclaimed in a given GCHeaderBuilder
+        # instance.
+        self.obj2header = {}
         self.size_gc_header = llmemory.GCHeaderOffset(self)
 
     def header_of_object(self, gcptr):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -281,6 +281,10 @@
         return lltype.Signed
 
 malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0)
+_translated_to_c = CDefinedIntSymbolic('1 /*_translated_to_c*/', default=0)
+
+def we_are_translated_to_c():
+    return we_are_translated() and _translated_to_c
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py
--- a/rpython/rlib/rdtoa.py
+++ b/rpython/rlib/rdtoa.py
@@ -56,22 +56,24 @@
         raise MemoryError
     end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
     try:
-        ll_input = rffi.str2charp(input)
+        # note: don't use the class scoped_view_charp here, it
+        # break some tests because this function is used by the GC
+        ll_input, flag = rffi.get_nonmovingbuffer_final_null(input)
         try:
             result = dg_strtod(ll_input, end_ptr)
 
             endpos = (rffi.cast(lltype.Signed, end_ptr[0]) -
                       rffi.cast(lltype.Signed, ll_input))
-
-            if endpos == 0 or endpos < len(input):
-                raise ValueError("invalid input at position %d" % (endpos,))
-
-            return result
         finally:
-            rffi.free_charp(ll_input)
+            rffi.free_nonmovingbuffer(input, ll_input, flag)
     finally:
         lltype.free(end_ptr, flavor='raw')
 
+    if endpos == 0 or endpos < len(input):
+        raise ValueError("invalid input at position %d" % (endpos,))
+
+    return result
+
 lower_special_strings = ['inf', '+inf', '-inf', 'nan']
 upper_special_strings = ['INF', '+INF', '-INF', 'NAN']
 
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1268,3 +1268,26 @@
     ptr = lltype.direct_arrayitems(array)
     # ptr is a Ptr(FixedSizeArray(Char, 1)).  Cast it to a rffi.CCHARP
     return rffi.cast(rffi.CCHARP, ptr)
+
+ at jit.dont_look_inside
+ at no_collect
+ at specialize.ll()
+def ll_write_final_null_char(s):
+    """'s' is a low-level STR; writes a terminating NULL character after
+    the other characters in 's'.  Warning, this only works because of
+    the 'extra_item_after_alloc' hack inside the definition of STR.
+    """
+    from rpython.rtyper.lltypesystem import rffi
+    PSTR = lltype.typeOf(s)
+    assert has_final_null_char(PSTR) == 1
+    n = llmemory.offsetof(PSTR.TO, 'chars')
+    n += llmemory.itemoffsetof(PSTR.TO.chars, 0)
+    n = llmemory.raw_malloc_usage(n)
+    n += len(s.chars)
+    # no GC operation from here!
+    ptr = rffi.cast(rffi.CCHARP, s)
+    ptr[n] = '\x00'
+
+ at specialize.memo()
+def has_final_null_char(PSTR):
+    return PSTR.TO.chars._hints.get('extra_item_after_alloc', 0)
diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py
--- a/rpython/rtyper/lltypesystem/ll2ctypes.py
+++ b/rpython/rtyper/lltypesystem/ll2ctypes.py
@@ -250,7 +250,9 @@
 
         if not A._hints.get('nolength'):
             _fields_ = [('length', lentype),
-                        ('items',  max_n * ctypes_item)]
+                        ('items',
+                           (max_n + A._hints.get('extra_item_after_alloc', 0))
+                           * ctypes_item)]
         else:
             _fields_ = [('items',  max_n * ctypes_item)]
 
@@ -695,6 +697,9 @@
         # we have no clue, so we allow whatever index
         return 0, maxint
 
+    def shrinklength(self, newlength):
+        raise NotImplementedError
+
     def getitem(self, index, uninitialized_ok=False):
         res = self._storage.contents._getitem(index, boundscheck=False)
         if isinstance(self._TYPE.OF, lltype.ContainerType):
diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py
--- a/rpython/rtyper/lltypesystem/llmemory.py
+++ b/rpython/rtyper/lltypesystem/llmemory.py
@@ -304,8 +304,15 @@
         return cast_ptr_to_adr(p)
 
     def raw_memcopy(self, srcadr, dstadr):
-        # should really copy the length field, but we can't
-        pass
+        # copy the length field, if we can
+        srclen = srcadr.ptr._obj.getlength()
+        dstlen = dstadr.ptr._obj.getlength()
+        if dstlen != srclen:
+            assert dstlen > srclen, "can't increase the length"
+            # a decrease in length occurs in the GC tests when copying a STR:
+            # the copy is initially allocated with really one extra char,
+            # the 'extra_item_after_alloc', and must be fixed.
+            dstadr.ptr._obj.shrinklength(srclen)
 
 
 class ArrayLengthOffset(AddressOffset):
@@ -390,11 +397,23 @@
     else:
         raise Exception("don't know how to take the size of a %r"%TYPE)
 
+ at specialize.memo()
+def extra_item_after_alloc(ARRAY):
+    assert isinstance(ARRAY, lltype.Array)
+    return ARRAY._hints.get('extra_item_after_alloc', 0)
+
 @specialize.arg(0)
 def sizeof(TYPE, n=None):
+    """Return the symbolic size of TYPE.
+    For a Struct with no varsized part, it must be called with n=None.
+    For an Array or a Struct with a varsized part, it is the number of items.
+    There is a special case to return 1 more than requested if the array
+    has the hint 'extra_item_after_alloc' set to 1.
+    """
     if n is None:
         return _sizeof_none(TYPE)
     elif isinstance(TYPE, lltype.Array):
+        n += extra_item_after_alloc(TYPE)
         return itemoffsetof(TYPE) + _sizeof_none(TYPE.OF) * n
     else:
         return _sizeof_int(TYPE, n)
@@ -1036,7 +1055,7 @@
                 _reccopy(subsrc, subdst)
             else:
                 # this is a hack XXX de-hack this
-                llvalue = source._obj.getitem(i, uninitialized_ok=True)
+                llvalue = source._obj.getitem(i, uninitialized_ok=2)
                 if not isinstance(llvalue, lltype._uninitialized):
                     dest._obj.setitem(i, llvalue)
     elif isinstance(T, lltype.Struct):
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1926,14 +1926,29 @@
         return 0, stop
 
     def getitem(self, index, uninitialized_ok=False):
-        v = self.items[index]
+        try:
+            v = self.items[index]
+        except IndexError:
+            if (index == len(self.items) and uninitialized_ok == 2 and
+                self._TYPE._hints.get('extra_item_after_alloc')):
+                # special case: reading the extra final char returns
+                # an uninitialized, if 'uninitialized_ok==2'
+                return _uninitialized(self._TYPE.OF)
+            raise
         if isinstance(v, _uninitialized) and not uninitialized_ok:
             raise UninitializedMemoryAccess("%r[%s]"%(self, index))
         return v
 
     def setitem(self, index, value):
         assert typeOf(value) == self._TYPE.OF
-        self.items[index] = value
+        try:
+            self.items[index] = value
+        except IndexError:
+            if (index == len(self.items) and value == '\x00' and
+                self._TYPE._hints.get('extra_item_after_alloc')):
+                # special case: writing NULL to the extra final char
+                return
+            raise
 
 assert not '__dict__' in dir(_array)
 assert not '__dict__' in dir(_struct)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -15,7 +15,7 @@
 from rpython.rtyper.tool.rfficache import platform, sizeof_c_type
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.annlowlevel import llhelper
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, we_are_translated_to_c
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0
 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import llmemory
@@ -232,40 +232,36 @@
             call_external_function = jit.dont_look_inside(
                 call_external_function)
 
+    def _oops():
+        raise AssertionError("can't pass (any more) a unicode string"
+                             " directly to a VOIDP argument")
+    _oops._annspecialcase_ = 'specialize:memo'
+
     unrolling_arg_tps = unrolling_iterable(enumerate(args))
     def wrapper(*args):
         real_args = ()
+        # XXX 'to_free' leaks if an allocation fails with MemoryError
+        # and was not the first in this function
         to_free = ()
         for i, TARGET in unrolling_arg_tps:
             arg = args[i]
-            freeme = None
-            if TARGET == CCHARP:
+            if TARGET == CCHARP or TARGET is VOIDP:
                 if arg is None:
                     arg = lltype.nullptr(CCHARP.TO)   # None => (char*)NULL
-                    freeme = arg
+                    to_free = to_free + (arg, '\x04')
                 elif isinstance(arg, str):
-                    arg = str2charp(arg)
-                    # XXX leaks if a str2charp() fails with MemoryError
-                    # and was not the first in this function
-                    freeme = arg
+                    tup = get_nonmovingbuffer_final_null(arg)
+                    to_free = to_free + tup
+                    arg = tup[0]
+                elif isinstance(arg, unicode):
+                    _oops()
             elif TARGET == CWCHARP:
                 if arg is None:
                     arg = lltype.nullptr(CWCHARP.TO)   # None => (wchar_t*)NULL
-                    freeme = arg
+                    to_free = to_free + (arg,)
                 elif isinstance(arg, unicode):
                     arg = unicode2wcharp(arg)
-                    # XXX leaks if a unicode2wcharp() fails with MemoryError
-                    # and was not the first in this function
-                    freeme = arg
-            elif TARGET is VOIDP:
-                if arg is None:
-                    arg = lltype.nullptr(VOIDP.TO)
-                elif isinstance(arg, str):
-                    arg = str2charp(arg)
-                    freeme = arg
-                elif isinstance(arg, unicode):
-                    arg = unicode2wcharp(arg)
-                    freeme = arg
+                    to_free = to_free + (arg,)
             elif _isfunctype(TARGET) and not _isllptr(arg):
                 # XXX pass additional arguments
                 use_gil = invoke_around_handlers
@@ -283,11 +279,22 @@
                            or TARGET is lltype.Bool)):
                         arg = cast(TARGET, arg)
             real_args = real_args + (arg,)
-            to_free = to_free + (freeme,)
         res = call_external_function(*real_args)
         for i, TARGET in unrolling_arg_tps:
-            if to_free[i]:
-                lltype.free(to_free[i], flavor='raw')
+            arg = args[i]
+            if TARGET == CCHARP or TARGET is VOIDP:
+                if arg is None:
+                    to_free = to_free[2:]
+                elif isinstance(arg, str):
+                    free_nonmovingbuffer(arg, to_free[0], to_free[1])
+                    to_free = to_free[2:]
+            elif TARGET == CWCHARP:
+                if arg is None:
+                    to_free = to_free[1:]
+                elif isinstance(arg, unicode):
+                    free_wcharp(to_free[0])
+                    to_free = to_free[1:]
+        assert len(to_free) == 0
         if rarithmetic.r_int is not r_int:
             if result is INT:
                 return cast(lltype.Signed, res)
@@ -816,52 +823,69 @@
         string is already nonmovable or could be pinned.  Must be followed by a
         free_nonmovingbuffer call.
 
-        First bool returned indicates if 'data' was pinned. Second bool returned
-        indicates if we did a raw alloc because pinning failed. Both bools
-        should never be true at the same time.
+        Also returns a char:
+         * \4: no pinning, returned pointer is inside 'data' which is nonmovable
+         * \5: 'data' was pinned, returned pointer is inside
+         * \6: pinning failed, returned pointer is raw malloced
+
+        For strings (not unicodes), the len()th character of the resulting
+        raw buffer is available, but not initialized.  Use
+        get_nonmovingbuffer_final_null() instead of get_nonmovingbuffer()
+        to get a regular null-terminated "char *".
         """
 
         lldata = llstrtype(data)
         count = len(data)
 
-        pinned = False
-        if rgc.can_move(data):
-            if rgc.pin(data):
-                pinned = True
+        if we_are_translated_to_c() and not rgc.can_move(data):
+            flag = '\x04'
+        else:
+            if we_are_translated_to_c() and rgc.pin(data):
+                flag = '\x05'
             else:
-                buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
+                buf = lltype.malloc(TYPEP.TO, count + (TYPEP is CCHARP),
+                                    flavor='raw')
                 copy_string_to_raw(lldata, buf, 0, count)
-                return buf, pinned, True
+                return buf, '\x06'
                 # ^^^ raw malloc used to get a nonmovable copy
         #
-        # following code is executed if:
+        # following code is executed after we're translated to C, if:
         # - rgc.can_move(data) and rgc.pin(data) both returned true
         # - rgc.can_move(data) returned false
         data_start = cast_ptr_to_adr(lldata) + \
             offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
 
-        return cast(TYPEP, data_start), pinned, False
+        return cast(TYPEP, data_start), flag
         # ^^^ already nonmovable. Therefore it's not raw allocated nor
         # pinned.
     get_nonmovingbuffer._always_inline_ = 'try' # get rid of the returned tuple
     get_nonmovingbuffer._annenforceargs_ = [strtype]
 
-    # (str, char*, bool, bool) -> None
+    @jit.dont_look_inside
+    def get_nonmovingbuffer_final_null(data):
+        tup = get_nonmovingbuffer(data)
+        buf, flag = tup
+        buf[len(data)] = lastchar
+        return tup
+    get_nonmovingbuffer_final_null._always_inline_ = 'try'
+    get_nonmovingbuffer_final_null._annenforceargs_ = [strtype]
+
+    # (str, char*, char) -> None
     # Can't inline this because of the raw address manipulation.
     @jit.dont_look_inside
-    def free_nonmovingbuffer(data, buf, is_pinned, is_raw):
+    def free_nonmovingbuffer(data, buf, flag):
         """
-        Keep 'data' alive and unpin it if it was pinned ('is_pinned' is true).
-        Otherwise free the non-moving copy ('is_raw' is true).
+        Keep 'data' alive and unpin it if it was pinned (flag==\5).
+        Otherwise free the non-moving copy (flag==\6).
         """
-        if is_pinned:
+        if flag == '\x05':
             rgc.unpin(data)
-        if is_raw:
+        if flag == '\x06':
             lltype.free(buf, flavor='raw')
-        # if is_pinned and is_raw are false: data was already nonmovable,
+        # if flag == '\x04': data was already nonmovable,
         # we have nothing to clean up
         keepalive_until_here(data)
-    free_nonmovingbuffer._annenforceargs_ = [strtype, None, bool, bool]
+    free_nonmovingbuffer._annenforceargs_ = [strtype, None, None]
 
     # int -> (char*, str, int)
     # Can't inline this because of the raw address manipulation.
@@ -947,18 +971,19 @@
 
     return (str2charp, free_charp, charp2str,
             get_nonmovingbuffer, free_nonmovingbuffer,
+            get_nonmovingbuffer_final_null,
             alloc_buffer, str_from_buffer, keep_buffer_alive_until_here,
             charp2strn, charpsize2str, str2chararray, str2rawmem,
             )
 
 (str2charp, free_charp, charp2str,
- get_nonmovingbuffer, free_nonmovingbuffer,
+ get_nonmovingbuffer, free_nonmovingbuffer, get_nonmovingbuffer_final_null,
  alloc_buffer, str_from_buffer, keep_buffer_alive_until_here,
  charp2strn, charpsize2str, str2chararray, str2rawmem,
  ) = make_string_mappings(str)
 
 (unicode2wcharp, free_wcharp, wcharp2unicode,
- get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer,
+ get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer, __not_usable,
  alloc_unicodebuffer, unicode_from_buffer, keep_unicodebuffer_alive_until_here,
  wcharp2unicoden, wcharpsize2unicode, unicode2wchararray, unicode2rawmem,
  ) = make_string_mappings(unicode)
@@ -1194,10 +1219,28 @@
     def __init__(self, data):
         self.data = data
     def __enter__(self):
-        self.buf, self.pinned, self.is_raw = get_nonmovingbuffer(self.data)
+        self.buf, self.flag = get_nonmovingbuffer(self.data)
         return self.buf
     def __exit__(self, *args):
-        free_nonmovingbuffer(self.data, self.buf, self.pinned, self.is_raw)
+        free_nonmovingbuffer(self.data, self.buf, self.flag)
+    __init__._always_inline_ = 'try'
+    __enter__._always_inline_ = 'try'
+    __exit__._always_inline_ = 'try'
+
+class scoped_view_charp:
+    """Returns a 'char *' that (tries to) point inside the given RPython
+    string (which must not be None).  You can replace scoped_str2charp()
+    with scoped_view_charp() in all places that guarantee that the
+    content of the 'char[]' array will not be modified.
+    """
+    def __init__(self, data):
+        self.data = data
+    __init__._annenforceargs_ = [None, annmodel.SomeString(can_be_None=False)]
+    def __enter__(self):
+        self.buf, self.flag = get_nonmovingbuffer_final_null(self.data)
+        return self.buf
+    def __exit__(self, *args):
+        free_nonmovingbuffer(self.data, self.buf, self.flag)
     __init__._always_inline_ = 'try'
     __enter__._always_inline_ = 'try'
     __exit__._always_inline_ = 'try'
@@ -1206,10 +1249,10 @@
     def __init__(self, data):
         self.data = data
     def __enter__(self):
-        self.buf, self.pinned, self.is_raw = get_nonmoving_unicodebuffer(self.data)
+        self.buf, self.flag = get_nonmoving_unicodebuffer(self.data)
         return self.buf
     def __exit__(self, *args):
-        free_nonmoving_unicodebuffer(self.data, self.buf, self.pinned, self.is_raw)
+        free_nonmoving_unicodebuffer(self.data, self.buf, self.flag)
     __init__._always_inline_ = 'try'
     __enter__._always_inline_ = 'try'
     __exit__._always_inline_ = 'try'
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -1238,7 +1238,8 @@
 # ____________________________________________________________
 
 STR.become(GcStruct('rpy_string', ('hash',  Signed),
-                    ('chars', Array(Char, hints={'immutable': True})),
+                    ('chars', Array(Char, hints={'immutable': True,
+                                    'extra_item_after_alloc': 1})),
                     adtmeths={'malloc' : staticAdtMethod(mallocstr),
                               'empty'  : staticAdtMethod(emptystrfun),
                               'copy_contents' : staticAdtMethod(copy_string_contents),
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -516,7 +516,7 @@
     def test_nonmovingbuffer(self):
         d = 'some cool data that should not move'
         def f():
-            buf, is_pinned, is_raw = get_nonmovingbuffer(d)
+            buf, flag = get_nonmovingbuffer(d)
             try:
                 counter = 0
                 for i in range(len(d)):
@@ -524,7 +524,7 @@
                         counter += 1
                 return counter
             finally:
-                free_nonmovingbuffer(d, buf, is_pinned, is_raw)
+                free_nonmovingbuffer(d, buf, flag)
         assert f() == len(d)
         fn = self.compile(f, [], gcpolicy='ref')
         assert fn() == len(d)
@@ -534,13 +534,13 @@
         def f():
             counter = 0
             for n in range(32):
-                buf, is_pinned, is_raw = get_nonmovingbuffer(d)
+                buf, flag = get_nonmovingbuffer(d)
                 try:
                     for i in range(len(d)):
                         if buf[i] == d[i]:
                             counter += 1
                 finally:
-                    free_nonmovingbuffer(d, buf, is_pinned, is_raw)
+                    free_nonmovingbuffer(d, buf, flag)
             return counter
         fn = self.compile(f, [], gcpolicy='semispace')
         # The semispace gc uses raw_malloc for its internal data structs
@@ -555,13 +555,13 @@
         def f():
             counter = 0
             for n in range(32):
-                buf, is_pinned, is_raw = get_nonmovingbuffer(d)
+                buf, flag = get_nonmovingbuffer(d)
                 try:
                     for i in range(len(d)):
                         if buf[i] == d[i]:
                             counter += 1
                 finally:
-                    free_nonmovingbuffer(d, buf, is_pinned, is_raw)
+                    free_nonmovingbuffer(d, buf, flag)
             return counter
         fn = self.compile(f, [], gcpolicy='incminimark')
         # The incminimark gc uses raw_malloc for its internal data structs
@@ -835,3 +835,11 @@
     if hasattr(rffi, '__INT128_T'):
         value = 0xAAAABBBBCCCCDDDD
         assert cast(rffi.__INT128_T, r_uint64(value)) == value
+
+def test_scoped_view_charp():
+    s = 'bar'
+    with scoped_view_charp(s) as buf:
+        assert buf[0] == 'b'
+        assert buf[1] == 'a'
+        assert buf[2] == 'r'
+        assert buf[3] == '\x00'
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -253,8 +253,11 @@
             yield '\t' + cdecl(typename, fname) + ';'
         if not self.ARRAY._hints.get('nolength', False):
             yield '\tlong length;'
+        varlength = self.varlength
+        if varlength is not None:
+            varlength += self.ARRAY._hints.get('extra_item_after_alloc', 0)
         line = '%s;' % cdecl(self.itemtypename,
-                             'items[%s]' % deflength(self.varlength))
+                             'items[%s]' % deflength(varlength))
         if self.ARRAY.OF is Void:    # strange
             line = '/* array of void */'
             if self.ARRAY._hints.get('nolength', False):
diff --git a/rpython/translator/c/src/precommondefs.h b/rpython/translator/c/src/precommondefs.h
--- a/rpython/translator/c/src/precommondefs.h
+++ b/rpython/translator/c/src/precommondefs.h
@@ -18,9 +18,9 @@
 #define _LARGEFILE_SOURCE 1
 /* Define on NetBSD to activate all library features */
 #define _NETBSD_SOURCE 1
-/* Define to activate features from IEEE Stds 1003.1-2001 */
+/* Define to activate features from IEEE Stds 1003.1-2008 */
 #ifndef _POSIX_C_SOURCE
-#  define _POSIX_C_SOURCE 200112L
+#  define _POSIX_C_SOURCE 200809L
 #endif
 /* Define on FreeBSD to activate all library features */
 #define __BSD_VISIBLE 1
diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py
--- a/rpython/translator/c/test/test_lltyped.py
+++ b/rpython/translator/c/test/test_lltyped.py
@@ -1,4 +1,4 @@
-import py
+import py, random
 from rpython.rtyper.lltypesystem.lltype import *
 from rpython.rtyper.lltypesystem import rffi
 from rpython.translator.c.test.test_genc import compile
@@ -255,28 +255,6 @@
         res2 = fn(0)
         assert res1 == res2
 
-    def test_null_padding(self):
-        py.test.skip("we no longer pad our RPython strings with a final NUL")
-        from rpython.rtyper.lltypesystem import llmemory
-        from rpython.rtyper.lltypesystem import rstr
-        chars_offset = llmemory.FieldOffset(rstr.STR, 'chars') + \
-                       llmemory.ArrayItemsOffset(rstr.STR.chars)
-        # sadly, there's no way of forcing this to fail if the strings
-        # are allocated in a region of memory such that they just
-        # happen to get a NUL byte anyway :/ (a debug build will
-        # always fail though)
-        def trailing_byte(s):
-            adr_s = llmemory.cast_ptr_to_adr(s)
-            return (adr_s + chars_offset).char[len(s)]
-        def f(x):
-            r = 0
-            for i in range(x):
-                r += ord(trailing_byte(' '*(100-x*x)))
-            return r
-        fn = self.getcompiled(f, [int])
-        res = fn(10)
-        assert res == 0
-
     def test_cast_primitive(self):
         def f(x):
             x = cast_primitive(UnsignedLongLong, x)
@@ -1023,3 +1001,49 @@
         assert fn(r_longlong(1)) == True
         assert fn(r_longlong(256)) == True
         assert fn(r_longlong(2**32)) == True
+
+    def test_extra_item_after_alloc(self):
+        from rpython.rlib import rgc
+        from rpython.rtyper.lltypesystem import lltype
+        from rpython.rtyper.lltypesystem import rstr
+        # all STR objects should be allocated with enough space for one
+        # extra char.  Check this for prebuilt strings, and for dynamically
+        # allocated ones with the default GC for tests.  Use strings of 8,
+        # 16 and 24 chars because if the extra char is missing, writing to it
+        # is likely to cause corruption in nearby structures.
+        sizes = [random.choice([8, 16, 24]) for i in range(100)]
+        A = lltype.Struct('A', ('x', lltype.Signed))
+        prebuilt = [(rstr.mallocstr(sz),
+                     lltype.malloc(A, flavor='raw', immortal=True))
+                        for sz in sizes]
+        k = 0
+        for i, (s, a) in enumerate(prebuilt):
+            a.x = i
+            for i in range(len(s.chars)):
+                k += 1
+                if k == 256:
+                    k = 1
+                s.chars[i] = chr(k)
+
+        def check(lst):
+            hashes = []
+            for i, (s, a) in enumerate(lst):
+                assert a.x == i
+                rgc.ll_write_final_null_char(s)
+            for i, (s, a) in enumerate(lst):
+                assert a.x == i     # check it was not overwritten
+        def f():
+            check(prebuilt)
+            lst1 = []
+            for i, sz in enumerate(sizes):
+                s = rstr.mallocstr(sz)
+                a = lltype.malloc(A, flavor='raw')
+                a.x = i
+                lst1.append((s, a))
+            check(lst1)
+            for _, a in lst1:
+                lltype.free(a, flavor='raw')
+            return 42
+
+        fn = self.getcompiled(f, [])
+        assert fn() == 42
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -3,6 +3,7 @@
 import os
 import sys
 import subprocess
+import random
 
 import py
 
@@ -1468,6 +1469,52 @@
         res = self.run('nursery_hash_base')
         assert res >= 195
 
+    def define_extra_item_after_alloc(cls):
+        from rpython.rtyper.lltypesystem import rstr
+        # all STR objects should be allocated with enough space for
+        # one extra char.  Check this with our GCs.  Use strings of 8,
+        # 16 and 24 chars because if the extra char is missing,
+        # writing to it is likely to cause corruption in nearby
+        # structures.
+        sizes = [random.choice([8, 16, 24]) for i in range(100)]
+        A = lltype.Struct('A', ('x', lltype.Signed))
+        prebuilt = [(rstr.mallocstr(sz),
+                     lltype.malloc(A, flavor='raw', immortal=True))
+                        for sz in sizes]
+        k = 0
+        for i, (s, a) in enumerate(prebuilt):
+            a.x = i
+            for i in range(len(s.chars)):
+                k += 1
+                if k == 256:
+                    k = 1
+                s.chars[i] = chr(k)
+
+        def check(lst):
+            hashes = []
+            for i, (s, a) in enumerate(lst):
+                assert a.x == i
+                rgc.ll_write_final_null_char(s)
+            for i, (s, a) in enumerate(lst):
+                assert a.x == i     # check it was not overwritten
+        def fn():
+            check(prebuilt)
+            lst1 = []
+            for i, sz in enumerate(sizes):
+                s = rstr.mallocstr(sz)
+                a = lltype.malloc(A, flavor='raw')
+                a.x = i
+                lst1.append((s, a))
+            check(lst1)
+            for _, a in lst1:
+                lltype.free(a, flavor='raw')
+            return 42
+        return fn
+
+    def test_extra_item_after_alloc(self):
+        res = self.run('extra_item_after_alloc')
+        assert res == 42
+
 
 class TestGenerationalGC(TestSemiSpaceGC):
     gcpolicy = "generation"
diff --git a/rpython/translator/tool/test/test_staticsizereport.py b/rpython/translator/tool/test/test_staticsizereport.py
--- a/rpython/translator/tool/test/test_staticsizereport.py
+++ b/rpython/translator/tool/test/test_staticsizereport.py
@@ -67,7 +67,7 @@
             (4 * S + 2 * P) +      # struct dicttable
             (S + 2 * 8192) +       # indexes, length 8192, rffi.USHORT
             (S + (S + S) * 3840) + # entries, length 3840
-            (S + S + 5) * 3840)    # 3840 strings with 5 chars each
+            (S + S + 6) * 3840)    # 3840 strings with 5 chars each (+1 final)
         assert guess_size(func.builder.db, fixarrayvalnode, set()) == 100 * rffi.sizeof(lltype.Signed) + 1 * rffi.sizeof(lltype.Signed)
         assert guess_size(func.builder.db, dynarrayvalnode, set()) == 100 * rffi.sizeof(lltype.Signed) + 2 * rffi.sizeof(lltype.Signed) + 1 * rffi.sizeof(rffi.VOIDP)
 

From pypy.commits at gmail.com  Wed Aug  3 08:00:48 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 03 Aug 2016 05:00:48 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: enforce that a successful
 vectorization in the pypy_c tests (micronumpy)
Message-ID: <57a1dcf0.919a1c0a.74348.6672@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86002:57316f5af6ff
Date: 2016-08-03 14:00 +0200
http://bitbucket.org/pypy/pypy/changeset/57316f5af6ff/

Log:	enforce that a successful vectorization in the pypy_c tests
	(micronumpy)

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -51,6 +51,10 @@
         log = self.run(main, [], vec=0)
         assert log.result == vlog.result
         assert log.result == result
+        assert log.jit_summary.vecopt_tried == 0
+        assert log.jit_summary.vecopt_success == 0
+        assert vlog.jit_summary.vecopt_tried > 0
+        assert vlog.jit_summary.vecopt_success > 0
 
 
     arith_comb = [
@@ -88,6 +92,10 @@
         log = self.run(main, [], vec=0)
         assert log.result == vlog.result
         assert log.result == result
+        assert log.jit_summary.vecopt_tried == 0
+        assert log.jit_summary.vecopt_success == 0
+        assert vlog.jit_summary.vecopt_tried > 0
+        assert vlog.jit_summary.vecopt_success > 0
 
     def test_reduce_logical_xor(self):
         def main():
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -148,7 +148,7 @@
         return info, loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
     except NotAVectorizeableLoop as e:
         debug_stop("vec-opt-loop")
-        debug_print("failed to vectorize loop. reason: %s" % str(e))
+        debug_print("failed to vectorize loop. reason: %s" % e.msg)
         # vectorization is not possible
         return loop_info, version.loop.finaloplist()
     except NotAProfitableLoop:

From pypy.commits at gmail.com  Wed Aug  3 08:44:38 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 03 Aug 2016 05:44:38 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: initializing the vector ext
 just after the cpu
Message-ID: <57a1e736.ca11c30a.76936.a97b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86003:768f96ea1752
Date: 2016-08-03 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/768f96ea1752/

Log:	initializing the vector ext just after the cpu

diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py
--- a/rpython/jit/backend/ppc/runner.py
+++ b/rpython/jit/backend/ppc/runner.py
@@ -8,7 +8,6 @@
 from rpython.jit.backend.ppc.arch import WORD
 from rpython.jit.backend.ppc.codebuilder import PPCBuilder
 from rpython.jit.backend.ppc import register as r
-from rpython.jit.backend.ppc.detect_feature import detect_vsx
 
 class PPC_CPU(AbstractLLCPU):
 
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -47,10 +47,6 @@
         or 'heap' not in enable_opts or 'pure' not in enable_opts):
         optimizations.append(OptSimplify(unroll))
 
-    cpu = metainterp_sd.cpu
-    if not cpu.vector_ext.is_setup():
-        cpu.vector_ext.setup_once(cpu.assembler)
-
     return optimizations, unroll
 
 def optimize_trace(metainterp_sd, jitdriver_sd, compile_data, memo=None):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1857,6 +1857,7 @@
             self.jitlog.setup_once()
             debug_print(self.jit_starting_line)
             self.cpu.setup_once()
+            self.cpu.vector_ext.setup_once(self.cpu.assembler)
             if not self.profiler.initialized:
                 self.profiler.start()
                 self.profiler.initialized = True

From pypy.commits at gmail.com  Wed Aug  3 09:39:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 03 Aug 2016 06:39:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: Work around a MSVC bug
Message-ID: <57a1f413.041f1c0a.8b27f.765b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86004:1512e98263d9
Date: 2016-08-03 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/1512e98263d9/

Log:	Work around a MSVC bug

diff --git a/rpython/translator/c/src/float.h b/rpython/translator/c/src/float.h
--- a/rpython/translator/c/src/float.h
+++ b/rpython/translator/c/src/float.h
@@ -34,10 +34,39 @@
 #define OP_CAST_FLOAT_TO_UINT(x,r)   r = (Unsigned)(x)
 #define OP_CAST_INT_TO_FLOAT(x,r)    r = (double)(x)
 #define OP_CAST_UINT_TO_FLOAT(x,r)   r = (double)(x)
-#define OP_CAST_LONGLONG_TO_FLOAT(x,r) r = (double)(x)
-#define OP_CAST_ULONGLONG_TO_FLOAT(x,r) r = (double)(x)
+#define OP_CAST_LONGLONG_TO_FLOAT(x,r) r = rpy_cast_longlong_to_float(x)
+#define OP_CAST_ULONGLONG_TO_FLOAT(x,r) r = rpy_cast_ulonglong_to_float(x)
 #define OP_CAST_BOOL_TO_FLOAT(x,r)   r = (double)(x)
 
+#ifdef _WIN32
+/* The purpose of these two functions is to work around a MSVC bug.
+   The expression '(double)131146795334735160LL' will lead to bogus
+   rounding, but apparently everything is fine if we write instead
+   rpy_cast_longlong_to_float(131146795334735160LL).  Tested with 
+   MSVC 2008.  Note that even if the two functions contain just
+   'return (double)x;' it seems to work on MSVC 2008, but I don't
+   trust that there are no other corner cases.
+   http://stackoverflow.com/questions/33829101/incorrect-double-to-long-conversion
+*/
+static _inline double rpy_cast_longlong_to_float(long long x)
+{
+    unsigned int lo = (unsigned int)x;
+    double result = lo;
+    result += ((int)(x >> 32)) * 4294967296.0;
+    return result;
+}
+static _inline double rpy_cast_ulonglong_to_float(unsigned long long x)
+{
+    unsigned int lo = (unsigned int)x;
+    double result = lo;
+    result += ((unsigned int)(x >> 32)) * 4294967296.0;
+    return result;
+}
+#else
+#  define rpy_cast_longlong_to_float(x) ((double)(x))
+#  define rpy_cast_ulonglong_to_float(x) ((double)(x))
+#endif
+
 #ifdef HAVE_LONG_LONG
 #define OP_CAST_FLOAT_TO_LONGLONG(x,r) r = (long long)(x)
 #define OP_CAST_FLOAT_TO_ULONGLONG(x,r) r = (unsigned long long)(x)

From pypy.commits at gmail.com  Wed Aug  3 12:21:57 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 03 Aug 2016 09:21:57 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Merge obsoleted impl of
 W_DictProxyObject into the correct one: fix __new__, __init__ and __repr__
Message-ID: <57a21a25.6814c30a.71dcd.faad@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r86005:fcb110da6372
Date: 2016-08-03 17:21 +0100
http://bitbucket.org/pypy/pypy/changeset/fcb110da6372/

Log:	Merge obsoleted impl of W_DictProxyObject into the correct one: fix
	__new__, __init__ and __repr__

diff --git a/pypy/objspace/std/classdict.py b/pypy/objspace/std/classdict.py
--- a/pypy/objspace/std/classdict.py
+++ b/pypy/objspace/std/classdict.py
@@ -9,35 +9,6 @@
 from pypy.objspace.std.typeobject import unwrap_cell
 
 
-class W_DictProxyObject(W_DictObject):
-    @staticmethod
-    def descr_new(space, w_type, w_mapping):
-        if (not space.lookup(w_mapping, "__getitem__") or
-            space.isinstance_w(w_mapping, space.w_list) or
-            space.isinstance_w(w_mapping, space.w_tuple)):
-            raise oefmt(space.w_TypeError,
-                        "mappingproxy() argument must be a mapping, not %T", w_mapping)
-        strategy = space.fromcache(MappingProxyStrategy)
-        storage = strategy.erase(w_mapping)
-        w_obj = space.allocate_instance(W_DictProxyObject, w_type)
-        W_DictProxyObject.__init__(w_obj, space, strategy, storage)
-        return w_obj
-
-    def descr_init(self, space, __args__):
-        pass
-
-    def descr_repr(self, space):
-        return space.wrap(u"mappingproxy(%s)" % (
-            space.unicode_w(W_DictObject.descr_repr(self, space))))
-
-W_DictProxyObject.typedef = TypeDef(
-    "mappingproxy", W_DictObject.typedef,
-    __new__ = interp2app(W_DictProxyObject.descr_new),
-    __init__ = interp2app(W_DictProxyObject.descr_init),
-    __repr__ = interp2app(W_DictProxyObject.descr_repr),
-)
-
-
 class ClassDictStrategy(DictStrategy):
     """Exposes a W_TypeObject.dict_w at app-level.
 
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -2,6 +2,7 @@
 # type.__dict__, so PyDictProxy_New has to use a custom read-only mapping.
 
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.interpreter.typedef import TypeDef, interp2app
 
@@ -11,6 +12,18 @@
     def __init__(self, w_mapping):
         self.w_mapping = w_mapping
 
+    @staticmethod
+    def descr_new(space, w_type, w_mapping):
+        if (not space.lookup(w_mapping, "__getitem__") or
+                space.isinstance_w(w_mapping, space.w_list) or
+                space.isinstance_w(w_mapping, space.w_tuple)):
+            raise oefmt(space.w_TypeError,
+                        "mappingproxy() argument must be a mapping, not %T", w_mapping)
+        return W_DictProxyObject(w_mapping)
+
+    def descr_init(self, space, __args__):
+        pass
+
     def descr_len(self, space):
         return space.len(self.w_mapping)
 
@@ -27,7 +40,8 @@
         return space.str(self.w_mapping)
 
     def descr_repr(self, space):
-        return space.repr(self.w_mapping)
+        return space.newunicode(u"mappingproxy(%s)" %
+                                (space.unicode_w(space.repr(self.w_mapping)),))
 
     @unwrap_spec(w_default=WrappedDefault(None))
     def get_w(self, space, w_key, w_default):
@@ -47,6 +61,8 @@
 
 W_DictProxyObject.typedef = TypeDef(
     'mappingproxy',
+    __new__=interp2app(W_DictProxyObject.descr_new),
+    __init__=interp2app(W_DictProxyObject.descr_init),
     __len__=interp2app(W_DictProxyObject.descr_len),
     __getitem__=interp2app(W_DictProxyObject.descr_getitem),
     __contains__=interp2app(W_DictProxyObject.descr_contains),

From pypy.commits at gmail.com  Wed Aug  3 13:47:32 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 03 Aug 2016 10:47:32 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Implement comparison methods for
 mappingproxy
Message-ID: <57a22e34.109a1c0a.ee027.40d3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r86006:361a24241f59
Date: 2016-08-03 18:47 +0100
http://bitbucket.org/pypy/pypy/changeset/361a24241f59/

Log:	Implement comparison methods for mappingproxy

diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -59,6 +59,19 @@
     def copy_w(self, space):
         return space.call_method(self.w_mapping, "copy")
 
+cmp_methods = {}
+def make_cmp_method(op):
+    def descr_op(self, space, w_other):
+        return getattr(space, op)(self.w_mapping, w_other)
+    descr_name = 'descr_' + op
+    descr_op.__name__ = descr_name
+    setattr(W_DictProxyObject, descr_name, descr_op)
+    cmp_methods['__%s__' % op] = interp2app(getattr(W_DictProxyObject, descr_name))
+
+for op in ['eq', 'ne', 'gt', 'ge', 'lt', 'le']:
+    make_cmp_method(op)
+
+
 W_DictProxyObject.typedef = TypeDef(
     'mappingproxy',
     __new__=interp2app(W_DictProxyObject.descr_new),
@@ -73,5 +86,6 @@
     keys=interp2app(W_DictProxyObject.keys_w),
     values=interp2app(W_DictProxyObject.values_w),
     items=interp2app(W_DictProxyObject.items_w),
-    copy=interp2app(W_DictProxyObject.copy_w)
+    copy=interp2app(W_DictProxyObject.copy_w),
+    **cmp_methods
 )

From pypy.commits at gmail.com  Wed Aug  3 15:42:38 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 03 Aug 2016 12:42:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Fix pyparser (now allows async and
 await as variable names)
Message-ID: <57a2492e.56421c0a.75916.57be@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86007:4ae7acb78ae8
Date: 2016-08-03 21:41 +0200
http://bitbucket.org/pypy/pypy/changeset/4ae7acb78ae8/

Log:	Fix pyparser (now allows async and await as variable names)

diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -96,7 +96,9 @@
     altindents = [0]
     last_comment = ''
     parenlevstart = (0, 0, "")
-    last_token = ''
+    async_def = False
+    async_def_nl = False
+    async_def_indent = 0
 
     # make the annotator happy
     endDFA = DUMMY_DFA
@@ -182,6 +184,10 @@
                     raise TokenIndentationError(err, line, lnum, 0, token_list)
                 if altcolumn != altindents[-1]:
                     raise TabError(lnum, pos, line)
+            if async_def_nl and async_def_indent >= indents[-1]:
+                async_def = False
+                async_def_nl = False
+                async_def_indent = 0
 
         else:                                  # continued statement
             if not line:
@@ -215,6 +221,8 @@
                     last_comment = ''
                 elif initial in '\r\n':
                     if parenlev <= 0:
+                        if async_def:
+                            async_def_nl = True
                         tok = (tokens.NEWLINE, last_comment, lnum, start, line)
                         token_list.append(tok)
                     last_comment = ''
@@ -254,10 +262,36 @@
                     if not verify_identifier(token):
                         raise TokenError("invalid character in identifier",
                                          line, lnum, start + 1, token_list)
-                    if token == 'async' and not last_token == 'def':
-                        token_list.append((tokens.ASYNC, token, lnum, start, line))
-                    elif token == 'await' and not last_token == 'def':
-                        token_list.append((tokens.AWAIT, token, lnum, start, line))
+
+                    if async_def:                          # inside 'async def' function
+                        if token == 'async':
+                            token_list.append((tokens.ASYNC, token, lnum, start, line))
+                        elif token == 'await':
+                            token_list.append((tokens.AWAIT, token, lnum, start, line))
+                        else:
+                            token_list.append((tokens.NAME, token, lnum, start, line))
+                    elif token == 'async':                 # async token, look ahead
+                        #ahead token
+                        if pos < max:
+                            as_pseudomatch = pseudoDFA.recognize(line, pos)
+                            as_start = whiteSpaceDFA.recognize(line, pos)
+                            if as_start < 0:
+                                as_start = pos
+                            as_end = as_pseudomatch
+            
+                            if as_start == as_end:
+                                raise TokenError("Unknown character", line,
+                                                 lnum, as_start + 1, token_list)
+            
+                            ahead_token = line[as_start:as_end]
+                            if ahead_token == 'def':
+                                async_def = True
+                                async_def_indent = indents[-1]
+                                token_list.append((tokens.ASYNC, token, lnum, start, line))
+                            else:
+                                token_list.append((tokens.NAME, token, lnum, start, line))
+                        else:
+                            token_list.append((tokens.NAME, token, lnum, start, line))
                     else:
                         token_list.append((tokens.NAME, token, lnum, start, line))
                     last_comment = ''
@@ -279,7 +313,6 @@
                         punct = tokens.OP
                     token_list.append((punct, token, lnum, start, line))
                     last_comment = ''
-                last_token = token
             else:
                 start = whiteSpaceDFA.recognize(line, pos)
                 if start < 0:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -175,9 +175,13 @@
         self.parse("await = 1")
         self.parse("def async(): pass")
         #async for
-        self.parse("async def foo(): async for a in b: pass")
+        self.parse("""async def foo():
+    async for a in b:
+        pass""")
         #async with
-        self.parse("async def foo(): async with a: pass")
+        self.parse("""async def foo():
+    async with a:
+        pass""")
 
 class TestPythonParserWithSpace:
 

From pypy.commits at gmail.com  Wed Aug  3 15:51:17 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 03 Aug 2016 12:51:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: More parser tests for async and
 await
Message-ID: <57a24b35.0675c20a.d6ef0.3c63@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86008:3f09fda522df
Date: 2016-08-03 21:50 +0200
http://bitbucket.org/pypy/pypy/changeset/3f09fda522df/

Log:	More parser tests for async and await

diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -170,6 +170,7 @@
 
     def test_async_await(self):
         self.parse("async def coro(): await func")
+        py.test.raises(SyntaxError, self.parse, 'await x')
         #Test as var and func name
         self.parse("async = 1")
         self.parse("await = 1")
@@ -178,10 +179,14 @@
         self.parse("""async def foo():
     async for a in b:
         pass""")
+        py.test.raises(SyntaxError, self.parse, 'def foo(): async for a in b: pass')
         #async with
         self.parse("""async def foo():
     async with a:
         pass""")
+        py.test.raises(SyntaxError, self.parse, 'def foo(): async with a: pass')
+        
+        
 
 class TestPythonParserWithSpace:
 
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -100,12 +100,6 @@
     async def foo():
         await await fut
     
-    await x
-    
-    def foo(): async for a in b: pass
-    
-    def foo(): async with a: pass
-    
 """)
 
 

From pypy.commits at gmail.com  Wed Aug  3 16:02:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 03 Aug 2016 13:02:24 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Record return from functions
 as events too
Message-ID: <57a24dd0.08d11c0a.daf7e.02f8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86009:84d843464c68
Date: 2016-08-03 22:04 +0200
http://bitbucket.org/pypy/pypy/changeset/84d843464c68/

Log:	Record return from functions as events too

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -87,9 +87,9 @@
                 # be accessed also later
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
-        if self.space.config.translation.reverse_debugger:
-            from pypy.interpreter.reverse_debugging import leave_call
-            leave_call(self.topframeref(), frame)
+            if self.space.config.translation.reverse_debugger:
+                from pypy.interpreter.reverse_debugging import leave_call
+                leave_call(self.topframeref(), frame)
 
     # ________________________________________________________________
 
diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -78,12 +78,14 @@
     if dbstate.breakpoint_stack_id != 0 and caller_frame is not None:
         if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame):
             revdb.breakpoint(-2)
+    if we_are_translated():
+        stop_point_activate(-2)
 
 
 def jump_backward(frame, jumpto):
     # When we see a jump backward, we set 'f_revdb_nextline_instr' in
     # such a way that the next instruction, at 'jumpto', will trigger
-    # stop_point_at_start_of_line().  We have to trigger it even if
+    # stop_point_activate().  We have to trigger it even if
     # 'jumpto' is not actually a start of line.  For example, after a
     # 'while foo:', the body ends with a JUMP_ABSOLUTE which
     # jumps back to the *second* opcode of the while.
@@ -117,12 +119,12 @@
         if ch == 0:
             pass   # we are at the start of a line now
         else:
-            # We are not, so don't call stop_point_at_start_of_line().
+            # We are not, so don't call stop_point_activate().
             # We still have to fill f_revdb_nextline_instr.
             call_stop_point_at_line = False
     #
     if call_stop_point_at_line:
-        stop_point_at_start_of_line()
+        stop_point_activate()
         cur += 1
         ch = ord(co_revdb_linestarts[cur])
     #
@@ -199,7 +201,7 @@
 non_standard_code = NonStandardCode()
 
 
-def stop_point_at_start_of_line():
+def stop_point_activate(place=0):
     if revdb.watch_save_state():
         any_watch_point = False
         space = dbstate.space
@@ -219,7 +221,7 @@
         revdb.watch_restore_state(any_watch_point)
         if watch_id != -1:
             revdb.breakpoint(watch_id)
-    revdb.stop_point()
+    revdb.stop_point(place)
 
 
 def load_metavar(index):
@@ -382,7 +384,8 @@
         indent))
     revdb.send_linecache(frame.getcode().co_filename, lineno)
 
-def display_function_part(frame, max_lines_before, max_lines_after):
+def display_function_part(frame, max_lines_before, max_lines_after,
+                          prompt="> "):
     code = frame.getcode()
     if code.co_filename.startswith('<builtin>'):
         return
@@ -400,7 +403,7 @@
     #
     for i in range(first_lineno, final_lineno + 1):
         if i == current_lineno:
-            revdb.send_output("> ")
+            revdb.send_output(prompt)
         else:
             revdb.send_output("  ")
         revdb.send_linecache(code.co_filename, i, strip=False)
@@ -415,7 +418,12 @@
     if cmd.c_arg1 == 0:
         revdb.send_output("%s:\n" % (
             file_and_lineno(frame, frame.get_last_lineno()),))
-        display_function_part(frame, max_lines_before=8, max_lines_after=5)
+        if revdb.current_place() == -2:
+            prompt = "<<"
+        else:
+            prompt = "> "
+        display_function_part(frame, max_lines_before=8, max_lines_after=5,
+                              prompt=prompt)
     elif cmd.c_arg1 == 2:
         display_function_part(frame, max_lines_before=1000,max_lines_after=1000)
     else:
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -26,14 +26,14 @@
 ANSWER_WATCH    = 23
 
 
-def stop_point():
+def stop_point(place=0):
     """Indicates a point in the execution of the RPython program where
     the reverse-debugger can stop.  When reverse-debugging, we see
     the "time" as the index of the stop-point that happened.
     """
     if we_are_translated():
         if fetch_translated_config().translation.reverse_debugger:
-            llop.revdb_stop_point(lltype.Void)
+            llop.revdb_stop_point(lltype.Void, place)
 
 def register_debug_command(command, lambda_func):
     """Register the extra RPython-implemented debug command."""
@@ -75,6 +75,12 @@
     unique id greater or equal."""
     return llop.revdb_get_value(lltype.SignedLongLong, 'u')
 
+def current_place():
+    """For RPython debug commands: the value of the 'place' argument
+    passed to stop_point().
+    """
+    return llop.revdb_get_value(lltype.Signed, 'p')
+
 ## @specialize.arg(1)
 ## def go_forward(time_delta, callback):
 ##     """For RPython debug commands: tells that after this function finishes,
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -145,14 +145,17 @@
 
 static void record_stop_point(void);
 static void replay_stop_point(void);
+static long current_place;
 
 RPY_EXTERN
-void rpy_reverse_db_stop_point(void)
+void rpy_reverse_db_stop_point(long place)
 {
     if (!RPY_RDB_REPLAY)
         record_stop_point();
-    else
+    else {
+        current_place = place;
         replay_stop_point();
+    }
 }
 
 
@@ -1244,6 +1247,8 @@
         return (flag_io_disabled == FID_REGULAR_MODE ?
                 rpy_revdb.unique_id_seen :
                 saved_state.unique_id_seen);
+    case 'p':       /* current_place() */
+        return current_place;
     default:
         return -1;
     }
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -129,9 +129,9 @@
         _RPY_REVDB_PRUID();                                             \
     } while (0)
 
-#define OP_REVDB_STOP_POINT(r)                                          \
+#define OP_REVDB_STOP_POINT(place, r)                                   \
     if (++rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)      \
-        rpy_reverse_db_stop_point()
+        rpy_reverse_db_stop_point(place)
 
 #define OP_REVDB_SEND_ANSWER(cmd, arg1, arg2, arg3, ll_string, r)       \
     rpy_reverse_db_send_answer(cmd, arg1, arg2, arg3, ll_string)
@@ -176,7 +176,7 @@
 
 RPY_EXTERN void rpy_reverse_db_flush(void);
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);
-RPY_EXTERN void rpy_reverse_db_stop_point(void);
+RPY_EXTERN void rpy_reverse_db_stop_point(long place);
 RPY_EXTERN void rpy_reverse_db_send_answer(int cmd, int64_t arg1, int64_t arg2,
                                            int64_t arg3, RPyString *extra);
 RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -341,6 +341,8 @@
             if extra == 'get-value':
                 revdb.send_answer(100, revdb.current_time(),
                                        revdb.total_time())
+            if extra == 'current-place':
+                revdb.send_answer(200, revdb.current_place())
             ## if extra == 'go-fw':
             ##     revdb.go_forward(1, went_fw)
             ## if cmdline == 'set-break-after-0':
@@ -375,7 +377,7 @@
             for i, op in enumerate(argv[1:]):
                 dbstate.stuff = Stuff()
                 dbstate.stuff.x = i + 1000
-                revdb.stop_point()
+                revdb.stop_point(i * 10)
                 print op
                 if i == 1:
                     if os.fork() == 0:    # child
@@ -419,6 +421,18 @@
         child.send(Message(1, extra='get-value'))
         child.expect(100, 1, 3)
 
+    def test_current_place(self):
+        child = self.replay()
+        child.send(Message(1, extra='current-place'))
+        child.expect(200, 0)
+        child.expect(42, 1, -43, -44, 'current-place')
+        child.expect(ANSWER_READY, 1, Ellipsis)
+        child.send(Message(CMD_FORWARD, 2))
+        child.expect(ANSWER_READY, 3, Ellipsis)
+        child.send(Message(1, extra='current-place'))
+        child.expect(200, 20)
+        child.expect(42, 1, -43, -44, 'current-place')
+
     ## def test_go_fw(self):
     ##     child = self.replay()
     ##     child.send(Message(1, extra='go-fw'))

From pypy.commits at gmail.com  Wed Aug  3 17:23:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 03 Aug 2016 14:23:47 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Add __pypy__.revdb_stop(),
 which makes an explicit breakpoint in revdb.
Message-ID: <57a260e3.a427c20a.88889.6999@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86010:88575763814a
Date: 2016-08-03 23:25 +0200
http://bitbucket.org/pypy/pypy/changeset/88575763814a/

Log:	Add __pypy__.revdb_stop(), which makes an explicit breakpoint in
	revdb. Use it in app_main to stop just after running the program
	(including if there was an exception).

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -79,10 +79,16 @@
     sys.stdout if needed, etc.
     """
     try:
+        from __pypy__ import revdb_stop
+    except ImportError:
+        revdb_stop = None
+    try:
         # run it
         try:
             f(*fargs, **fkwds)
         finally:
+            if revdb_stop:
+                revdb_stop()
             sys.settrace(None)
             sys.setprofile(None)
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -89,7 +89,7 @@
             jit.virtual_ref_finish(frame_vref, frame)
             if self.space.config.translation.reverse_debugger:
                 from pypy.interpreter.reverse_debugging import leave_call
-                leave_call(self.topframeref(), frame)
+                leave_call(self.topframeref(), got_exception)
 
     # ________________________________________________________________
 
diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -74,12 +74,16 @@
     if code.co_revdb_linestarts is None:
         build_co_revdb_linestarts(code)
 
-def leave_call(caller_frame, callee_frame):
+def leave_call(caller_frame, got_exception):
     if dbstate.breakpoint_stack_id != 0 and caller_frame is not None:
         if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame):
             revdb.breakpoint(-2)
     if we_are_translated():
-        stop_point_activate(-2)
+        stop_point_activate(-2 + got_exception)
+
+def stop_point():
+    if we_are_translated():
+        revdb.breakpoint(-3)
 
 
 def jump_backward(frame, jumpto):
@@ -418,10 +422,12 @@
     if cmd.c_arg1 == 0:
         revdb.send_output("%s:\n" % (
             file_and_lineno(frame, frame.get_last_lineno()),))
-        if revdb.current_place() == -2:
-            prompt = "<<"
+        if revdb.current_place() == -2:   # <= this is the arg to stop_point()
+            prompt = "<<"     # return
+        elif revdb.current_place() == -1:
+            prompt = "!!"     # exceptional return
         else:
-            prompt = "> "
+            prompt = "> "     # plain line
         display_function_part(frame, max_lines_before=8, max_lines_after=5,
                               prompt=prompt)
     elif cmd.c_arg1 == 2:
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -129,3 +129,6 @@
             features = detect_cpu.getcpufeatures(model)
             self.extra_interpdef('jit_backend_features',
                                     'space.wrap(%r)' % features)
+        if self.space.config.translation.reverse_debugger:
+            self.extra_interpdef('revdb_stop',
+                                 'interp_magic.revdb_stop')
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -204,3 +204,7 @@
     after changing the Python code.
     """
     return space.wrap(space._side_effects_ok())
+
+def revdb_stop(space):
+    from pypy.interpreter.reverse_debugging import stop_point
+    stop_point()
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -134,6 +134,9 @@
         elif break_at[0] == 'W':
             kind = 'watchpoint'
             name = self.pgroup.all_breakpoints.sources.get(num, '??')
+        elif num == -3:
+            kind = 'stoppoint'
+            name = 'explicit stop'
         else:
             kind = '?????point'
             name = repr(break_at)
@@ -191,9 +194,11 @@
         printing = []
         for num in b.regular_breakpoint_nums():
             kind, name = self._bp_kind(num)
-            printing.append('%s %s %d: %s' % (
+            printing.append('%s %s%s: %s' % (
                 'Reverse-hit' if backward else 'Hit',
-                kind, num, name))
+                kind,
+                '' if kind == 'stoppoint' else ' %d' % (num,),
+                name))
         self.print_extra_pending_info = '\n'.join(printing)
         if self.pgroup.get_current_time() != b.time:
             target_time = b.time

From pypy.commits at gmail.com  Thu Aug  4 06:45:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 04 Aug 2016 03:45:49 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57a31cdd.2624c20a.7a5a1.499c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86012:bd6d62fca9de
Date: 2016-08-04 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/bd6d62fca9de/

Log:	merge default

diff --git a/rpython/translator/c/src/float.h b/rpython/translator/c/src/float.h
--- a/rpython/translator/c/src/float.h
+++ b/rpython/translator/c/src/float.h
@@ -34,10 +34,39 @@
 #define OP_CAST_FLOAT_TO_UINT(x,r)   r = (Unsigned)(x)
 #define OP_CAST_INT_TO_FLOAT(x,r)    r = (double)(x)
 #define OP_CAST_UINT_TO_FLOAT(x,r)   r = (double)(x)
-#define OP_CAST_LONGLONG_TO_FLOAT(x,r) r = (double)(x)
-#define OP_CAST_ULONGLONG_TO_FLOAT(x,r) r = (double)(x)
+#define OP_CAST_LONGLONG_TO_FLOAT(x,r) r = rpy_cast_longlong_to_float(x)
+#define OP_CAST_ULONGLONG_TO_FLOAT(x,r) r = rpy_cast_ulonglong_to_float(x)
 #define OP_CAST_BOOL_TO_FLOAT(x,r)   r = (double)(x)
 
+#ifdef _WIN32
+/* The purpose of these two functions is to work around a MSVC bug.
+   The expression '(double)131146795334735160LL' will lead to bogus
+   rounding, but apparently everything is fine if we write instead
+   rpy_cast_longlong_to_float(131146795334735160LL).  Tested with 
+   MSVC 2008.  Note that even if the two functions contain just
+   'return (double)x;' it seems to work on MSVC 2008, but I don't
+   trust that there are no other corner cases.
+   http://stackoverflow.com/questions/33829101/incorrect-double-to-long-conversion
+*/
+static _inline double rpy_cast_longlong_to_float(long long x)
+{
+    unsigned int lo = (unsigned int)x;
+    double result = lo;
+    result += ((int)(x >> 32)) * 4294967296.0;
+    return result;
+}
+static _inline double rpy_cast_ulonglong_to_float(unsigned long long x)
+{
+    unsigned int lo = (unsigned int)x;
+    double result = lo;
+    result += ((unsigned int)(x >> 32)) * 4294967296.0;
+    return result;
+}
+#else
+#  define rpy_cast_longlong_to_float(x) ((double)(x))
+#  define rpy_cast_ulonglong_to_float(x) ((double)(x))
+#endif
+
 #ifdef HAVE_LONG_LONG
 #define OP_CAST_FLOAT_TO_LONGLONG(x,r) r = (long long)(x)
 #define OP_CAST_FLOAT_TO_ULONGLONG(x,r) r = (unsigned long long)(x)

From pypy.commits at gmail.com  Thu Aug  4 06:45:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 04 Aug 2016 03:45:47 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: moved setup_once for the vector
 extension, debug_print when NotAVectorizableLoop is raised
Message-ID: <57a31cdb.4675c20a.6c840.4bf3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86011:613b2a914410
Date: 2016-08-04 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/613b2a914410/

Log:	moved setup_once for the vector extension, debug_print when
	NotAVectorizableLoop is raised

diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py
--- a/rpython/jit/backend/llsupport/vector_ext.py
+++ b/rpython/jit/backend/llsupport/vector_ext.py
@@ -8,6 +8,9 @@
         failnbail_transformation)
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.debug import debug_print
 
 class TypeRestrict(object):
     ANY_TYPE = '\x00'
@@ -192,11 +195,11 @@
                 continue
             curvecinfo = forwarded_vecinfo(arg)
             if curvecinfo.bytesize != bytesize:
-                raise NotAVectorizeableLoop("op match size first type failed %d != %d" % \
-                        (curvecinfo.bytesize != bytesize))
+                debug_print("op match size first type failed")
+                raise NotAVectorizeableLoop
             if curvecinfo.datatype != datatype:
-                raise NotAVectorizeableLoop("op match size first type failed (datatype). %s != %s" % \
-                        (curvecinfo.datatype != datatype))
+                debug_print("op match size first type failed (datatype)")
+                raise NotAVectorizeableLoop
         return None
 
 TR_ANY = TypeRestrict()
diff --git a/rpython/jit/metainterp/jitexc.py b/rpython/jit/metainterp/jitexc.py
--- a/rpython/jit/metainterp/jitexc.py
+++ b/rpython/jit/metainterp/jitexc.py
@@ -62,10 +62,8 @@
             self.red_int, self.red_ref, self.red_float)
 
 class NotAVectorizeableLoop(JitException):
-    def __init__(self, msg="<empty>"):
-        self.msg = msg
     def __str__(self):
-        return 'NotAVectorizeableLoop(%s)' % self.msg
+        return 'NotAVectorizeableLoop()'
 
 class NotAProfitableLoop(JitException):
     def __str__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -26,6 +26,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
 from rpython.rlib.jit import Counters
+from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.jit.backend.llsupport.symbolic import (WORD as INT_WORD,
         SIZEOF_FLOAT as FLOAT_WORD)
@@ -146,9 +147,8 @@
         #
         info.label_op = loop.label
         return info, loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
-    except NotAVectorizeableLoop as e:
+    except NotAVectorizeableLoop:
         debug_stop("vec-opt-loop")
-        debug_print("failed to vectorize loop. reason: %s" % e.msg)
         # vectorization is not possible
         return loop_info, version.loop.finaloplist()
     except NotAProfitableLoop:
@@ -160,8 +160,6 @@
         debug_stop("vec-opt-loop")
         debug_print("failed to vectorize loop. THIS IS A FATAL ERROR!")
         if we_are_translated():
-            from rpython.rtyper.lltypesystem import lltype
-            from rpython.rtyper.lltypesystem.lloperation import llop
             llop.debug_print_traceback(lltype.Void)
         else:
             raise
@@ -230,12 +228,17 @@
         self.linear_find_smallest_type(loop)
         byte_count = self.smallest_type_bytes
         vsize = self.vector_ext.vec_size()
-        if vsize == 0 or byte_count == 0 or loop.label.getopnum() != rop.LABEL:
-            # stop, there is no chance to vectorize this trace
+        # stop, there is no chance to vectorize this trace
             # we cannot optimize normal traces (if there is no label)
-            raise NotAVectorizeableLoop("vsize %d byte_count %d not label? %d" % \
-                    (vsize, byte_count, loop.label.getopnum() != rop.LABEL))
-
+        if vsize == 0:
+            debug_print("vector size is zero")
+            raise NotAVectorizeableLoop
+        if byte_count == 0:
+            debug_print("could not find smallest type")
+            raise NotAVectorizeableLoop
+        if loop.label.getopnum() != rop.LABEL:
+            debug_print("not a loop, can only vectorize loops")
+            raise NotAVectorizeableLoop
         # find index guards and move to the earliest position
         graph = self.analyse_index_calculations(loop)
         if graph is not None:
@@ -432,7 +435,8 @@
             intersecting edges.
         """
         if len(self.packset.packs) == 0:
-            raise NotAVectorizeableLoop("packset is empty")
+            debug_print("packset is empty")
+            raise NotAVectorizeableLoop
         i = 0
         j = 0
         end_ij = len(self.packset.packs)
@@ -664,7 +668,8 @@
                 if forward and origin_pack.is_accumulating():
                     # in this case the splitted accumulator must
                     # be combined. This case is not supported
-                    raise NotAVectorizeableLoop("splitted accum must be flushed here (not supported)")
+                    debug_print("splitted accum must be flushed here (not supported)")
+                    raise NotAVectorizeableLoop
                 #
                 if self.contains_pair(lnode, rnode):
                     return None
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1857,7 +1857,8 @@
             self.jitlog.setup_once()
             debug_print(self.jit_starting_line)
             self.cpu.setup_once()
-            self.cpu.vector_ext.setup_once(self.cpu.assembler)
+            if self.cpu.vector_ext:
+                self.cpu.vector_ext.setup_once(self.cpu.assembler)
             if not self.profiler.initialized:
                 self.profiler.start()
                 self.profiler.initialized = True

From pypy.commits at gmail.com  Thu Aug  4 11:15:26 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 04 Aug 2016 08:15:26 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Fix tests for compatibility with
 CPython, which makes them pass on this branch as well
Message-ID: <57a35c0e.262ec20a.1424d.b174@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r86013:2467b1a9d1ba
Date: 2016-08-04 16:14 +0100
http://bitbucket.org/pypy/pypy/changeset/2467b1a9d1ba/

Log:	Fix tests for compatibility with CPython, which makes them pass on
	this branch as well

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -970,7 +970,6 @@
         raises(TypeError, setattr, list, 'foobar', 42)
         raises(TypeError, delattr, dict, 'keys')
         raises(TypeError, 'int.__dict__["a"] = 1')
-        raises(TypeError, 'int.__dict__.clear()')
 
     def test_nontype_in_mro(self):
         class OldStyle:
@@ -1028,10 +1027,9 @@
             pass
 
         a = A()
+        d = A.__dict__
         A.x = 1
-        assert A.__dict__["x"] == 1
-        A.__dict__['x'] = 5
-        assert A.x == 5
+        assert d["x"] == 1
 
     def test_we_already_got_one_1(self):
         # Issue #2079: highly obscure: CPython complains if we say

From pypy.commits at gmail.com  Thu Aug  4 11:30:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 04 Aug 2016 08:30:50 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Do not attempt to modify a class
 dict, use setattr instead
Message-ID: <57a35faa.c3881c0a.e7298.7567@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r86014:4244afa6cdb0
Date: 2016-08-04 16:29 +0100
http://bitbucket.org/pypy/pypy/changeset/4244afa6cdb0/

Log:	Do not attempt to modify a class dict, use setattr instead

diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -175,7 +175,7 @@
          "__new__"      : make_new(class_name),
          }
     pycppclass = metacpp(class_name, _drop_cycles(bases), d)
- 
+
     # cache result early so that the class methods can find the class itself
     setattr(scope, final_class_name, pycppclass)
 
@@ -192,13 +192,10 @@
     for dm_name in cppclass.get_datamember_names():
         cppdm = cppclass.get_datamember(dm_name)
 
-        # here, setattr() can not be used, because a data member can shadow one in
-        # its base class, resulting in the __set__() of its base class being called
-        # by setattr(); so, store directly on the dictionary
-        pycppclass.__dict__[dm_name] = cppdm
+        setattr(pycppclass, dm_name, cppdm)
         import cppyy
         if cppyy._is_static(cppdm):     # TODO: make this a method of cppdm
-            metacpp.__dict__[dm_name] = cppdm
+            setattr(metacpp, dm_name, cppdm)
 
     # the call to register will add back-end specific pythonizations and thus
     # needs to run first, so that the generic pythonizations can use them
@@ -413,7 +410,7 @@
         lib = cppyy._load_dictionary(name)
         _loaded_dictionaries[name] = lib
         return lib
-    
+
 def _init_pythonify():
     # cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause cppyy to be loaded

From pypy.commits at gmail.com  Thu Aug  4 12:06:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 04 Aug 2016 09:06:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2363: extracted a unit test that
 fails
Message-ID: <57a36809.a427c20a.88889.cbb3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86015:1d29fd069ef0
Date: 2016-08-04 17:40 +0200
http://bitbucket.org/pypy/pypy/changeset/1d29fd069ef0/

Log:	Issue #2363: extracted a unit test that fails

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -1,6 +1,7 @@
 import py
 from rpython.rtyper.lltypesystem import lltype, llmemory, llarena
 from rpython.memory.gc.incminimark import IncrementalMiniMarkGC, WORD
+from rpython.memory.gc.incminimark import GCFLAG_VISITED
 from test_direct import BaseDirectGCTest
 
 T = lltype.GcForwardReference()
@@ -981,3 +982,56 @@
         self.gc.major_collection_step()   # should not crash reading 'ptr1'!
 
         del self.gc.TEST_VISIT_SINGLE_STEP
+
+
+    def test_pin_bug2(self):
+        #
+        # * we have an old object A that points to a pinned object B
+        #
+        # * we unpin B
+        #
+        # * the next minor_collection() is done in STATE_MARKING==1
+        #   when the object A is already black
+        #
+        # * _minor_collection() => _visit_old_objects_pointing_to_pinned()
+        #   which will move the now-unpinned B out of the nursery, to B'
+        #
+        # At that point we need to take care of colors, otherwise we
+        # get a black object (A) pointing to a white object (B'),
+        # which must never occur.
+        #
+        ptrA = self.malloc(T)
+        ptrA.someInt = 42
+        adrA = llmemory.cast_ptr_to_adr(ptrA)
+        res = self.gc.pin(adrA)
+        assert res
+
+        ptrC = self.malloc(S)
+        self.stackroots.append(ptrC)
+
+        ptrB = self.malloc(S)
+        ptrB.data = ptrA
+        self.stackroots.append(ptrB)
+
+        self.gc.collect()
+        ptrB = self.stackroots[-1]    # now old and outside the nursery
+        ptrC = self.stackroots[-2]    # another random old object, traced later
+        adrB = llmemory.cast_ptr_to_adr(ptrB)
+
+        self.gc.minor_collection()
+        assert self.gc.gc_state == self.STATE_SCANNING
+        self.gc.major_collection_step()
+        assert self.gc.gc_state == self.STATE_MARKING
+        assert not (self.gc.header(adrB).tid & GCFLAG_VISITED)  # not black yet
+
+        self.gc.TEST_VISIT_SINGLE_STEP = True
+        self.gc.major_collection_step()
+        assert self.gc.gc_state == self.STATE_MARKING
+        assert self.gc.header(adrB).tid & GCFLAG_VISITED    # now black
+        # but ptrC is not traced yet, which is why we're still in STATE_MARKING
+        assert self.gc.old_objects_pointing_to_pinned.tolist() == [adrB]
+
+        self.gc.unpin(adrA)
+
+        self.gc.DEBUG = 2
+        self.gc.minor_collection()

From pypy.commits at gmail.com  Thu Aug  4 12:06:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 04 Aug 2016 09:06:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2363: fix. The issue was a
 "black->white" rare case in the
Message-ID: <57a3680b.2916c20a.891ef.cc79@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86016:93a5d95ec126
Date: 2016-08-04 18:06 +0200
http://bitbucket.org/pypy/pypy/changeset/93a5d95ec126/

Log:	Issue #2363: fix. The issue was a "black->white" rare case in the
	incremental GC, which is forbidden.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1633,6 +1633,14 @@
             # have been modified and need rescanning.
             self.old_objects_pointing_to_young.foreach(
                 self._add_to_more_objects_to_trace, None)
+            # Old black objects pointing to pinned objects that may no
+            # longer be pinned now: careful,
+            # _visit_old_objects_pointing_to_pinned() will move the
+            # previously-pinned object, and that creates a white object.
+            # We prevent the "black->white" situation by forcing the
+            # old black object to become gray again.
+            self.old_objects_pointing_to_pinned.foreach(
+                self._add_to_more_objects_to_trace_if_black, None)
         #
         # First, find the roots that point to young objects.  All nursery
         # objects found are copied out of the nursery, and the occasional
@@ -2144,6 +2152,10 @@
         self.header(obj).tid &= ~GCFLAG_VISITED
         self.more_objects_to_trace.append(obj)
 
+    def _add_to_more_objects_to_trace_if_black(self, obj, ignored):
+        if self.header(obj).tid & GCFLAG_VISITED:
+            self._add_to_more_objects_to_trace(obj, ignored)
+
     def minor_and_major_collection(self):
         # First, finish the current major gc, if there is one in progress.
         # This is a no-op if the gc_state is already STATE_SCANNING.

From pypy.commits at gmail.com  Thu Aug  4 12:08:41 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 04 Aug 2016 09:08:41 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: encode the jitdrivers name in
 start_trace, jitlog version bump
Message-ID: <57a36889.c15e1c0a.917d6.80a7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86017:74b2ed7be48b
Date: 2016-08-04 16:32 +0200
http://bitbucket.org/pypy/pypy/changeset/74b2ed7be48b/

Log:	encode the jitdrivers name in start_trace, jitlog version bump

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1051,8 +1051,9 @@
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     #
+    jd_name = jitdriver_sd.jitdriver.name
     metainterp_sd.jitlog.start_new_trace(metainterp_sd,
-            faildescr=resumekey, entry_bridge=False)
+            faildescr=resumekey, entry_bridge=False, jd_name=jd_name)
     #
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 1
+JITLOG_VERSION = 2
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -331,7 +331,7 @@
     def finish(self):
         jitlog_teardown()
 
-    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False):
+    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False, jd_name=""):
         # even if the logger is not enabled, increment the trace id
         self.trace_id += 1
         if not jitlog_enabled():
@@ -345,6 +345,7 @@
         else:
             content.append(encode_str('loop'))
             content.append(encode_le_addr(int(entry_bridge)))
+        content.append(encode_str(jd_name))
         self._write_marked(MARK_START_TRACE, ''.join(content))
 
     def trace_aborted(self):

From pypy.commits at gmail.com  Thu Aug  4 12:08:43 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 04 Aug 2016 09:08:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: translation fixes batch 1
Message-ID: <57a3688b.c15e1c0a.917d6.80ab@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async
Changeset: r86018:d108f2d1bd0d
Date: 2016-08-04 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/d108f2d1bd0d/

Log:	translation fixes batch 1

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -668,6 +668,8 @@
 
     # TODO 
     ops.BUILD_LIST_FROM_ARG: 1,
+    # TODO
+    ops.LOAD_CLASSDEREF: 1,
 }
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -273,17 +273,11 @@
                     elif token == 'async':                 # async token, look ahead
                         #ahead token
                         if pos < max:
-                            as_pseudomatch = pseudoDFA.recognize(line, pos)
-                            as_start = whiteSpaceDFA.recognize(line, pos)
-                            if as_start < 0:
-                                as_start = pos
-                            as_end = as_pseudomatch
-            
-                            if as_start == as_end:
-                                raise TokenError("Unknown character", line,
-                                                 lnum, as_start + 1, token_list)
-            
-                            ahead_token = line[as_start:as_end]
+                            async_end = pseudoDFA.recognize(line, pos)
+                            assert async_end >= 3
+                            async_start = async_end - 3
+                            assert async_start >= 0
+                            ahead_token = line[async_start:async_end]
                             if ahead_token == 'def':
                                 async_def = True
                                 async_def_indent = indents[-1]
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -226,10 +226,11 @@
             size = rffi.sizeof(rffi.VOIDP)
         return size
     
-    def descr_cast(self, space, w_args, **w_kwds):
-        self._check_released(space)
-        newitemsize = self.get_native_fmtchar(w_args._val(w_args))
-        mv = W_MemoryView(self.buf, w_args._val(w_args), newitemsize)
+    def descr_cast(self, space, w_args, w_kwds):
+        # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
+        #self._check_released(space)
+        #newitemsize = self.get_native_fmtchar(w_args._val(w_args))
+        return W_MemoryView(self.buf, self.format, self.itemsize)
         return mv
 
 

From pypy.commits at gmail.com  Thu Aug  4 13:29:22 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 04 Aug 2016 10:29:22 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Add failing cpyext test clarifying
 the expected behaviour of type->tp_dict
Message-ID: <57a37b72.45c8c20a.3d264.4834@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r86019:332fe6be115d
Date: 2016-08-04 18:28 +0100
http://bitbucket.org/pypy/pypy/changeset/332fe6be115d/

Log:	Add failing cpyext test clarifying the expected behaviour of
	type->tp_dict

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -277,9 +277,23 @@
                      args->ob_type->tp_dict, "copy");
                  Py_INCREF(method);
                  return method;
-             ''')])
+             '''),
+            ("get_type_dict", "METH_O",
+             '''
+                PyObject* value = args->ob_type->tp_dict;
+                if (value == NULL) value = Py_None;
+                Py_INCREF(value);
+                return value;
+             '''),
+            ])
         obj = foo.new()
         assert module.read_tp_dict(obj) == foo.fooType.copy
+        assert type(module.get_type_dict(obj)) is dict
+        d = module.get_type_dict(1)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert int._some_attribute == 1
+        del d["_some_attribute"]
 
     def test_custom_allocation(self):
         foo = self.import_module("foo")

From pypy.commits at gmail.com  Thu Aug  4 16:09:16 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 04 Aug 2016 13:09:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: add failing test - cpyext uses a
 buffered FILE* but W_File uses a non-bufferd int fid
Message-ID: <57a3a0ec.041f1c0a.342ac.d227@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86021:700c5913e7ea
Date: 2016-08-04 21:35 +0300
http://bitbucket.org/pypy/pypy/changeset/700c5913e7ea/

Log:	add failing test - cpyext uses a buffered FILE* but W_File uses a
	non-bufferd int fid

diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -1,6 +1,6 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers, fdopen)
+    cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers, c_fdopen)
 from pypy.module.cpyext.pyobject import PyObject
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from pypy.interpreter.error import (OperationError, oefmt, 
@@ -64,7 +64,7 @@
     if (fd < 0 or not mode or mode[0] not in ['r', 'w', 'a', 'U'] or
         ('U' in mode and ('w' in mode or 'a' in mode))):
         raise oefmt(space.w_IOError, 'invalid fileno or mode') 
-    ret = fdopen(fd, mode)
+    ret = c_fdopen(fd, mode)
     if not ret:
         raise exception_from_saved_errno(space, space.w_IOError)
     return ret
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,4 @@
 from pypy.conftest import option
-from pypy.module.cpyext.api import fopen, fclose, fwrite
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
@@ -133,6 +132,15 @@
                     return PyLong_FromLong(0);
                 return PyLong_FromLong(ftell(fp));
              """),
+            ("read_10", "METH_O",
+             """
+                char s[10];
+                FILE * fp = PyFile_AsFile(args);
+                if (fp == NULL)
+                    return PyLong_FromLong(0);
+                fread(s, 1, 10, fp);
+                return PyLong_FromLong(ftell(fp));
+             """),
             ])
         filename = self.udir + "/_test_file"
         with open(filename, 'w') as fid:
@@ -142,5 +150,12 @@
             t_py = fid.tell()
             assert t_py == 80
             t_c = module.get_c_tell(fid)
-        assert t_c == t_py
+            assert t_c == t_py
+            print '-------- tell ',t_c
+            t_c = module.read_10(fid)
+            assert t_c == t_py + 10
+            print '-------- tell ',t_c
+            t_py = fid.tell()
+            assert t_c == t_py, 'after a fread, c level ftell(fp) %d but PyFile.tell() %d' % (t_c, t_py)
 
+

From pypy.commits at gmail.com  Thu Aug  4 16:09:18 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 04 Aug 2016 13:09:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: refactor - reuse externals from rfile
Message-ID: <57a3a0ee.53b81c0a.e11d7.d2ea@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86022:3c04fe2c5184
Date: 2016-08-04 21:36 +0300
http://bitbucket.org/pypy/pypy/changeset/3c04fe2c5184/

Log:	refactor - reuse externals from rfile

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,6 +11,9 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+        c_fdopen, c_fileno,
+        c_fopen)# for tests
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import NameManager
@@ -85,44 +88,32 @@
 assert CONST_WSTRING == rffi.CWCHARP
 
 # FILE* interface
-FILEP = rffi.COpaquePtr('FILE')
 
 if sys.platform == 'win32':
     dash = '_'
 else:
     dash = ''
-fileno = rffi.llexternal(dash + 'fileno', [FILEP], rffi.INT)
-fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
-fdopen = rffi.llexternal(dash + 'fdopen', [rffi.INT, CONST_STRING],
-                  FILEP, save_err=rffi.RFFI_SAVE_ERRNO)
 
-_fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
-    if not is_valid_fd(fileno(fp)):
+    if not is_valid_fd(c_fileno(fp)):
         return -1
-    return _fclose(fp)
+    return c_fclose(fp)
 
-_fwrite = rffi.llexternal('fwrite',
-                         [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                         rffi.SIZE_T)
 def fwrite(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fwrite(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fwrite(buf, sz, n, fp)
 
-_fread = rffi.llexternal('fread',
-                        [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                        rffi.SIZE_T)
 def fread(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fread(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fread(buf, sz, n, fp)
 
 _feof = rffi.llexternal('feof', [FILEP], rffi.INT)
 def feof(fp):
-    validate_fd(fileno(fp))
+    validate_fd(c_fileno(fp))
     return _feof(fp)
 
 def is_valid_fp(fp):
-    return is_valid_fd(fileno(fp))
+    return is_valid_fd(c_fileno(fp))
 
 pypy_decl = 'pypy_decl.h'
 
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.eval import (
     Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP
+from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
@@ -130,19 +130,19 @@
     def test_run_file(self, space, api):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
-        fp = fopen(str(filepath), "rb")
+        fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_ZeroDivisionError
         api.PyErr_Clear()
 
         # try again, but with a closed file
-        fp = fopen(str(filepath), "rb")
-        os.close(fileno(fp))
+        fp = c_fopen(str(filepath), "rb")
+        os.close(c_fileno(fp))
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 

From pypy.commits at gmail.com  Thu Aug  4 16:09:19 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 04 Aug 2016 13:09:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix for 700c5913e7ea - force unbuffered
 FILE* use
Message-ID: <57a3a0ef.cb7f1c0a.515df.cf21@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86023:01f46383f3ce
Date: 2016-08-04 21:37 +0300
http://bitbucket.org/pypy/pypy/changeset/01f46383f3ce/

Log:	fix for 700c5913e7ea - force unbuffered FILE* use

diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -1,4 +1,5 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rfile import c_setvbuf, _IONBF
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers, c_fdopen)
 from pypy.module.cpyext.pyobject import PyObject
@@ -67,8 +68,9 @@
     ret = c_fdopen(fd, mode)
     if not ret:
         raise exception_from_saved_errno(space, space.w_IOError)
+    # XXX fix this once use-file-star-for-file lands
+    c_setvbuf(ret, lltype.nullptr(rffi.CCHARP.TO), _IONBF, 0)
     return ret
-        
 
 @cpython_api([FILEP, CONST_STRING, CONST_STRING, rffi.VOIDP], PyObject)
 def PyFile_FromFile(space, fp, name, mode, close):

From pypy.commits at gmail.com  Fri Aug  5 02:30:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 04 Aug 2016 23:30:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: Found out why we couldn't call
 _add_to_more_objects_to_trace_*if_black*
Message-ID: <57a43285.2916c20a.891ef.a995@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86024:9adb8eec4ac2
Date: 2016-08-05 08:32 +0200
http://bitbucket.org/pypy/pypy/changeset/9adb8eec4ac2/

Log:	Found out why we couldn't call
	_add_to_more_objects_to_trace_*if_black* here. Potential obscure fix
	for other cases too.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1632,7 +1632,7 @@
             # This is because these are precisely the old objects that
             # have been modified and need rescanning.
             self.old_objects_pointing_to_young.foreach(
-                self._add_to_more_objects_to_trace, None)
+                self._add_to_more_objects_to_trace_if_black, None)
             # Old black objects pointing to pinned objects that may no
             # longer be pinned now: careful,
             # _visit_old_objects_pointing_to_pinned() will move the
@@ -2269,6 +2269,11 @@
             if (not self.objects_to_trace.non_empty() and
                 not self.more_objects_to_trace.non_empty()):
                 #
+                # First, 'prebuilt_root_objects' might have grown since
+                # we scanned it in collect_roots() (rare case).  Rescan.
+                self.collect_nonstack_roots()
+                self.visit_all_objects()
+                #
                 if self.rrc_enabled:
                     self.rrc_major_collection_trace()
                 #
@@ -2449,21 +2454,30 @@
         return nobjects
 
 
-    def collect_roots(self):
-        # Collect all roots.  Starts from all the objects
-        # from 'prebuilt_root_objects'.
+    def collect_nonstack_roots(self):
+        # Non-stack roots: first, the objects from 'prebuilt_root_objects'
         self.prebuilt_root_objects.foreach(self._collect_obj, None)
         #
-        # Add the roots from the other sources.
+        # Add the roots from static prebuilt non-gc structures
         self.root_walker.walk_roots(
-            IncrementalMiniMarkGC._collect_ref_stk, # stack roots
-            IncrementalMiniMarkGC._collect_ref_stk, # static in prebuilt non-gc structures
+            None,
+            IncrementalMiniMarkGC._collect_ref_stk,
             None)   # we don't need the static in all prebuilt gc objects
         #
         # If we are in an inner collection caused by a call to a finalizer,
         # the 'run_finalizers' objects also need to be kept alive.
         self.enum_pending_finalizers(self._collect_obj, None)
 
+    def collect_roots(self):
+        # Collect all roots.  Starts from the non-stack roots.
+        self.collect_nonstack_roots()
+        #
+        # Add the stack roots.
+        self.root_walker.walk_roots(
+            IncrementalMiniMarkGC._collect_ref_stk, # stack roots
+            None,
+            None)
+
     def enumerate_all_roots(self, callback, arg):
         self.prebuilt_root_objects.foreach(callback, arg)
         MovingGCBase.enumerate_all_roots(self, callback, arg)

From pypy.commits at gmail.com  Fri Aug  5 04:00:52 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 05 Aug 2016 01:00:52 -0700 (PDT)
Subject: [pypy-commit] pypy resource_warning: merge default
Message-ID: <57a447b4.c5aa1c0a.db8ad.7d07@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: resource_warning
Changeset: r86025:9f2557766bf3
Date: 2016-08-05 10:00 +0200
http://bitbucket.org/pypy/pypy/changeset/9f2557766bf3/

Log:	merge default

diff too long, truncating to 2000 out of 77823 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -20,3 +20,10 @@
 5f8302b8bf9f53056e40426f10c72151564e5b19 release-4.0.1
 246c9cf22037b11dc0e8c29ce3f291d3b8c5935a release-5.0
 bbd45126bc691f669c4ebdfbd74456cd274c6b92 release-5.0.1
+3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1
+b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1
+80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -43,17 +43,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -93,9 +93,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -104,30 +104,34 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
-  Stefano Rivera
   Mark Pearse
   Simon Cross
+  Edd Barrett
   Andreas St&#252;hrk
-  Edd Barrett
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Spenser Bauman
   Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
-  Spenser Bauman
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -139,7 +143,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Tobias Pape
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -155,11 +158,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -172,7 +177,7 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
-  Devin Jeanpierre
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -182,7 +187,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -206,18 +210,17 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
-  Mark Young
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -225,6 +228,7 @@
   Pieter Zieschang
   Gabriel
   Lukas Vacek
+  Kunal Grover
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -240,7 +244,6 @@
   Kristjan Valur Jonsson
   David Lievens
   Neil Blakey-Milner
-  Sergey Matyunin
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
@@ -252,9 +255,11 @@
   Artur Lisiecki
   Sergey Kishchenko
   Ignas Mikalajunas
+  Alecsandru Patrascu
   Christoph Gerum
   Martin Blais
   Lene Wagner
+  Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
   Toni Mattis
@@ -266,8 +271,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -293,6 +299,7 @@
   Stephan Busemann
   Rafa&#322; Ga&#322;czy&#324;ski
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
@@ -305,6 +312,7 @@
   Boglarka Vezer
   Chris Pressey
   Buck Golemon
+  Diana Popa
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -85,10 +85,11 @@
     pass
 
 def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')):
+    import ast
     result = []
     for word in re_word.findall(line):
         if word.startswith('"'):
-            word = eval(word)
+            word = ast.literal_eval(word)
         result.append(word)
     return result
 
diff --git a/dotviewer/graphserver.py b/dotviewer/graphserver.py
--- a/dotviewer/graphserver.py
+++ b/dotviewer/graphserver.py
@@ -143,6 +143,11 @@
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:
+        if len(sys.argv) == 1:
+            # start locally
+            import sshgraphserver
+            sshgraphserver.ssh_graph_server(['LOCAL'])
+            sys.exit(0)
         print >> sys.stderr, __doc__
         sys.exit(2)
     if sys.argv[1] == '--stdio':
diff --git a/dotviewer/sshgraphserver.py b/dotviewer/sshgraphserver.py
--- a/dotviewer/sshgraphserver.py
+++ b/dotviewer/sshgraphserver.py
@@ -4,11 +4,14 @@
 
 Usage:
     sshgraphserver.py  hostname  [more args for ssh...]
+    sshgraphserver.py  LOCAL
 
 This logs in to 'hostname' by passing the arguments on the command-line
 to ssh.  No further configuration is required: it works for all programs
 using the dotviewer library as long as they run on 'hostname' under the
 same username as the one sshgraphserver logs as.
+
+If 'hostname' is the string 'LOCAL', then it starts locally without ssh.
 """
 
 import graphserver, socket, subprocess, random
@@ -18,12 +21,19 @@
     s1 = socket.socket()
     s1.bind(('127.0.0.1', socket.INADDR_ANY))
     localhost, localport = s1.getsockname()
-    remoteport = random.randrange(10000, 20000)
-    #  ^^^ and just hope there is no conflict
 
-    args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)]
-    args = args + sshargs + ['python -u -c "exec input()"']
-    print ' '.join(args[:-1])
+    if sshargs[0] != 'LOCAL':
+        remoteport = random.randrange(10000, 20000)
+        #  ^^^ and just hope there is no conflict
+
+        args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (
+            remoteport, localport)]
+        args = args + sshargs + ['python -u -c "exec input()"']
+    else:
+        remoteport = localport
+        args = ['python', '-u', '-c', 'exec input()']
+
+    print ' '.join(args)
     p = subprocess.Popen(args, bufsize=0,
                          stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE)
diff --git a/lib-python/2.7/distutils/cmd.py b/lib-python/2.7/distutils/cmd.py
--- a/lib-python/2.7/distutils/cmd.py
+++ b/lib-python/2.7/distutils/cmd.py
@@ -298,8 +298,16 @@
         src_cmd_obj.ensure_finalized()
         for (src_option, dst_option) in option_pairs:
             if getattr(self, dst_option) is None:
-                setattr(self, dst_option,
-                        getattr(src_cmd_obj, src_option))
+                try:
+                    setattr(self, dst_option,
+                            getattr(src_cmd_obj, src_option))
+                except AttributeError:
+                    # This was added after problems with setuptools 18.4.
+                    # It seems that setuptools 20.9 fixes the problem.
+                    # But e.g. on Ubuntu 14.04 with /usr/bin/virtualenv
+                    # if I say "virtualenv -p pypy venv-pypy" then it
+                    # just installs setuptools 18.4 from some cache...
+                    pass
 
 
     def get_finalized_command(self, command, create=1):
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -834,54 +834,63 @@
             c2pread, c2pwrite = None, None
             errread, errwrite = None, None
 
+            ispread = False
             if stdin is None:
                 p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
                 if p2cread is None:
                     p2cread, _ = _subprocess.CreatePipe(None, 0)
+                    ispread = True
             elif stdin == PIPE:
                 p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
+                ispread = True
             elif isinstance(stdin, int):
                 p2cread = msvcrt.get_osfhandle(stdin)
             else:
                 # Assuming file-like object
                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
-            p2cread = self._make_inheritable(p2cread)
+            p2cread = self._make_inheritable(p2cread, ispread)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(p2cread)
             if stdin == PIPE:
                 to_close.add(p2cwrite)
 
+            ispwrite = False
             if stdout is None:
                 c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
                     _, c2pwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stdout == PIPE:
                 c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif isinstance(stdout, int):
                 c2pwrite = msvcrt.get_osfhandle(stdout)
             else:
                 # Assuming file-like object
                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-            c2pwrite = self._make_inheritable(c2pwrite)
+            c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(c2pwrite)
             if stdout == PIPE:
                 to_close.add(c2pread)
 
+            ispwrite = False
             if stderr is None:
                 errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
                 if errwrite is None:
                     _, errwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif stderr == STDOUT:
-                errwrite = c2pwrite.handle # pass id to not close it
+                errwrite = c2pwrite
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
                 # Assuming file-like object
                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
-            errwrite = self._make_inheritable(errwrite)
+            errwrite = self._make_inheritable(errwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(errwrite)
             if stderr == PIPE:
@@ -892,13 +901,14 @@
                     errread, errwrite), to_close
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
             dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
-            # If the initial handle was obtained with CreatePipe, close it.
-            if not isinstance(handle, int):
+            # PyPy: If the initial handle was obtained with CreatePipe,
+            # close it.
+            if close:
                 handle.Close()
             return dupl
 
diff --git a/lib-python/2.7/test/test_descr.py b/lib-python/2.7/test/test_descr.py
--- a/lib-python/2.7/test/test_descr.py
+++ b/lib-python/2.7/test/test_descr.py
@@ -1735,7 +1735,6 @@
             ("__reversed__", reversed, empty_seq, set(), {}),
             ("__length_hint__", list, zero, set(),
              {"__iter__" : iden, "next" : stop}),
-            ("__sizeof__", sys.getsizeof, zero, set(), {}),
             ("__instancecheck__", do_isinstance, return_true, set(), {}),
             ("__missing__", do_dict_missing, some_number,
              set(("__class__",)), {}),
@@ -1747,6 +1746,8 @@
             ("__format__", format, format_impl, set(), {}),
             ("__dir__", dir, empty_seq, set(), {}),
             ]
+        if test_support.check_impl_detail():
+            specials.append(("__sizeof__", sys.getsizeof, zero, set(), {}))
 
         class Checker(object):
             def __getattr__(self, attr, test=self):
@@ -1768,10 +1769,6 @@
                 raise MyException
 
         for name, runner, meth_impl, ok, env in specials:
-            if name == '__length_hint__' or name == '__sizeof__':
-                if not test_support.check_impl_detail():
-                    continue
-
             class X(Checker):
                 pass
             for attr, obj in env.iteritems():
diff --git a/lib-python/2.7/test/test_hash.py b/lib-python/2.7/test/test_hash.py
--- a/lib-python/2.7/test/test_hash.py
+++ b/lib-python/2.7/test/test_hash.py
@@ -174,7 +174,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 
diff --git a/lib-python/2.7/test/test_sys_settrace.py b/lib-python/2.7/test/test_sys_settrace.py
--- a/lib-python/2.7/test/test_sys_settrace.py
+++ b/lib-python/2.7/test/test_sys_settrace.py
@@ -328,8 +328,8 @@
 
     def test_13_genexp(self):
         if self.using_gc:
+            gc.enable()
             test_support.gc_collect()
-            gc.enable()
         try:
             self.run_test(generator_example)
             # issue1265: if the trace function contains a generator,
diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
--- a/lib-python/stdlib-upgrade.txt
+++ b/lib-python/stdlib-upgrade.txt
@@ -5,15 +5,23 @@
 
     overly detailed
 
-1. check out the branch vendor/stdlib
+0. make sure your working dir is clean
+1. check out the branch vendor/stdlib (for 2.7) or vendor/stdlib-3-* (for py3k)
+   or create branch vendor/stdlib-3-*
 2. upgrade the files there
+   2a. remove lib-python/2.7/ or lib-python/3/
+   2b. copy the files from the cpython repo
+   2c. hg add lib-python/2.7/ or lib-python/3/
+   2d. hg remove --after
+   2e. show copied files in cpython repo by running `hg diff --git -r v<old> -r v<new> Lib | grep '^copy \(from\|to\)'`
+   2f. fix copies / renames manually by running `hg copy --after <from> <to>` for each copied file
 3. update stdlib-version.txt with the output of hg -id from the cpython repo
 4. commit
-5. update to default/py3k
+5. update to default / py3k
 6. create a integration branch for the new stdlib
    (just hg branch stdlib-$version)
-7. merge vendor/stdlib
+7. merge vendor/stdlib or vendor/stdlib-3-*
 8. commit
 10. fix issues
 11. commit --close-branch
-12. merge to default
+12. merge to default / py3k
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -320,8 +320,7 @@
     def __reduce_ex__(self, proto):
         return type(self), (list(self), self.maxlen)
 
-    def __hash__(self):
-        raise TypeError("deque objects are unhashable")
+    __hash__ = None
 
     def __copy__(self):
         return self.__class__(self, self.maxlen)
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -199,10 +199,13 @@
     return tp._alignmentofinstances()
 
 @builtinify
-def byref(cdata):
+def byref(cdata, offset=0):
     # "pointer" is imported at the end of this module to avoid circular
     # imports
-    return pointer(cdata)
+    ptr = pointer(cdata)
+    if offset != 0:
+        ptr._buffer[0] += offset
+    return ptr
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -6,7 +6,7 @@
 irc_header = "And now for something completely different"
 
 
-def interactive_console(mainmodule=None, quiet=False):
+def interactive_console(mainmodule=None, quiet=False, future_flags=0):
     # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
     # mimics what CPython does in pythonrun.c
     if not hasattr(sys, 'ps1'):
@@ -37,15 +37,17 @@
             raise ImportError
         from pyrepl.simple_interact import run_multiline_interactive_console
     except ImportError:
-        run_simple_interactive_console(mainmodule)
+        run_simple_interactive_console(mainmodule, future_flags=future_flags)
     else:
-        run_multiline_interactive_console(mainmodule)
+        run_multiline_interactive_console(mainmodule, future_flags=future_flags)
 
-def run_simple_interactive_console(mainmodule):
+def run_simple_interactive_console(mainmodule, future_flags=0):
     import code
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
     # some parts of code.py are copied here because it seems to be impossible
     # to start an interactive console without printing at least one line
     # of banner
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -224,23 +224,9 @@
 va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat <psobym>
 """
 
-from string import ascii_uppercase, ascii_lowercase
-
 def rot13(data):
-    """ A simple rot-13 encoder since `str.encode('rot13')` was removed from
-        Python as of version 3.0.  It rotates both uppercase and lowercase letters individually.
-    """
-    total = []
-    for char in data:
-        if char in ascii_uppercase:
-            index = (ascii_uppercase.find(char) + 13) % 26
-            total.append(ascii_uppercase[index])
-        elif char in ascii_lowercase:
-            index = (ascii_lowercase.find(char) + 13) % 26
-            total.append(ascii_lowercase[index])
-        else:
-            total.append(char)
-    return "".join(total)
+    return ''.join(chr(ord(c)+(13 if 'A'<=c.upper()<='M' else
+                              -13 if 'N'<=c.upper()<='Z' else 0)) for c in data)
 
 def some_topic():
     import time
diff --git a/lib_pypy/_pypy_wait.py b/lib_pypy/_pypy_wait.py
--- a/lib_pypy/_pypy_wait.py
+++ b/lib_pypy/_pypy_wait.py
@@ -1,51 +1,22 @@
-from resource import _struct_rusage, struct_rusage
-from ctypes import CDLL, c_int, POINTER, byref
-from ctypes.util import find_library
+from resource import ffi, lib, _make_struct_rusage
 
 __all__ = ["wait3", "wait4"]
 
-libc = CDLL(find_library("c"))
-c_wait3 = libc.wait3
-c_wait3.argtypes = [POINTER(c_int), c_int, POINTER(_struct_rusage)]
-c_wait3.restype = c_int
-
-c_wait4 = libc.wait4
-c_wait4.argtypes = [c_int, POINTER(c_int), c_int, POINTER(_struct_rusage)]
-c_wait4.restype = c_int
-
-def create_struct_rusage(c_struct):
-    return struct_rusage((
-        float(c_struct.ru_utime),
-        float(c_struct.ru_stime),
-        c_struct.ru_maxrss,
-        c_struct.ru_ixrss,
-        c_struct.ru_idrss,
-        c_struct.ru_isrss,
-        c_struct.ru_minflt,
-        c_struct.ru_majflt,
-        c_struct.ru_nswap,
-        c_struct.ru_inblock,
-        c_struct.ru_oublock,
-        c_struct.ru_msgsnd,
-        c_struct.ru_msgrcv,
-        c_struct.ru_nsignals,
-        c_struct.ru_nvcsw,
-        c_struct.ru_nivcsw))
 
 def wait3(options):
-    status = c_int()
-    _rusage = _struct_rusage()
-    pid = c_wait3(byref(status), c_int(options), byref(_rusage))
+    status = ffi.new("int *")
+    ru = ffi.new("struct rusage *")
+    pid = lib.wait3(status, options, ru)
 
-    rusage = create_struct_rusage(_rusage)
+    rusage = _make_struct_rusage(ru)
 
-    return pid, status.value, rusage
+    return pid, status[0], rusage
 
 def wait4(pid, options):
-    status = c_int()
-    _rusage = _struct_rusage()
-    pid = c_wait4(c_int(pid), byref(status), c_int(options), byref(_rusage))
+    status = ffi.new("int *")
+    ru = ffi.new("struct rusage *")
+    pid = lib.wait4(pid, status, options, ru)
 
-    rusage = create_struct_rusage(_rusage)
+    rusage = _make_struct_rusage(ru)
 
-    return pid, status.value, rusage
+    return pid, status[0], rusage
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -0,0 +1,91 @@
+# Note: uses the CFFI out-of-line ABI mode.  We can't use the API
+# mode because ffi.compile() needs to run the compiler, which
+# needs 'subprocess', which needs 'msvcrt' and '_subprocess',
+# which depend on '_pypy_winbase_cffi' already.
+#
+# Note that if you need to regenerate _pypy_winbase_cffi and
+# can't use a preexisting PyPy to do that, then running this
+# file should work as long as 'subprocess' is not imported
+# by cffi.  I had to hack in 'cffi._pycparser' to move an
+#'import subprocess' to the inside of a function.  (Also,
+# CPython+CFFI should work as well.)
+#
+# This module supports both msvcrt.py and _subprocess.py.
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pypy_winbase_cffi", None)
+
+# ---------- MSVCRT ----------
+
+ffi.cdef("""
+typedef unsigned short wint_t;
+
+int _open_osfhandle(intptr_t osfhandle, int flags);
+intptr_t _get_osfhandle(int fd);
+int _setmode(int fd, int mode);
+int _locking(int fd, int mode, long nbytes);
+
+int _kbhit(void);
+int _getch(void);
+wint_t _getwch(void);
+int _getche(void);
+wint_t _getwche(void);
+int _putch(int);
+wint_t _putwch(wchar_t);
+int _ungetch(int);
+wint_t _ungetwch(wint_t);
+""")
+
+# ---------- SUBPROCESS ----------
+
+ffi.cdef("""
+typedef struct {
+    DWORD  cb;
+    char * lpReserved;
+    char * lpDesktop;
+    char * lpTitle;
+    DWORD  dwX;
+    DWORD  dwY;
+    DWORD  dwXSize;
+    DWORD  dwYSize;
+    DWORD  dwXCountChars;
+    DWORD  dwYCountChars;
+    DWORD  dwFillAttribute;
+    DWORD  dwFlags;
+    WORD   wShowWindow;
+    WORD   cbReserved2;
+    LPBYTE lpReserved2;
+    HANDLE hStdInput;
+    HANDLE hStdOutput;
+    HANDLE hStdError;
+} STARTUPINFO, *LPSTARTUPINFO;
+
+typedef struct {
+    HANDLE hProcess;
+    HANDLE hThread;
+    DWORD  dwProcessId;
+    DWORD  dwThreadId;
+} PROCESS_INFORMATION, *LPPROCESS_INFORMATION;
+
+DWORD WINAPI GetVersion(void);
+BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD);
+BOOL WINAPI CloseHandle(HANDLE);
+HANDLE WINAPI GetCurrentProcess(void);
+BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
+                            DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
+BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
+BOOL WINAPI TerminateProcess(HANDLE, UINT);
+HANDLE WINAPI GetStdHandle(DWORD);
+""")
+
+# --------------------
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -0,0 +1,10 @@
+# auto-generated file
+import _cffi_backend
+
+ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
+    _version = 0x2601,
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+)
diff --git a/lib_pypy/_resource_build.py b/lib_pypy/_resource_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_resource_build.py
@@ -0,0 +1,118 @@
+from cffi import FFI
+
+ffi = FFI()
+
+# Note: we don't directly expose 'struct timeval' or 'struct rlimit'
+
+
+rlimit_consts = '''
+RLIMIT_CPU
+RLIMIT_FSIZE
+RLIMIT_DATA
+RLIMIT_STACK
+RLIMIT_CORE
+RLIMIT_NOFILE
+RLIMIT_OFILE
+RLIMIT_VMEM
+RLIMIT_AS
+RLIMIT_RSS
+RLIMIT_NPROC
+RLIMIT_MEMLOCK
+RLIMIT_SBSIZE
+RLIM_INFINITY
+RUSAGE_SELF
+RUSAGE_CHILDREN
+RUSAGE_BOTH
+'''.split()
+
+rlimit_consts = ['#ifdef %s\n\t{"%s", %s},\n#endif\n' % (s, s, s)
+                 for s in rlimit_consts]
+
+
+ffi.set_source("_resource_cffi", """
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <sys/wait.h>
+
+static const struct my_rlimit_def {
+    const char *name;
+    long long value;
+} my_rlimit_consts[] = {
+$RLIMIT_CONSTS
+    { NULL, 0 }
+};
+
+#define doubletime(TV) ((double)(TV).tv_sec + (TV).tv_usec * 0.000001)
+
+static double my_utime(struct rusage *input)
+{
+    return doubletime(input->ru_utime);
+}
+
+static double my_stime(struct rusage *input)
+{
+    return doubletime(input->ru_stime);
+}
+
+static int my_getrlimit(int resource, long long result[2])
+{
+    struct rlimit rl;
+    if (getrlimit(resource, &rl) == -1)
+        return -1;
+    result[0] = rl.rlim_cur;
+    result[1] = rl.rlim_max;
+    return 0;
+}
+
+static int my_setrlimit(int resource, long long cur, long long max)
+{
+    struct rlimit rl;
+    rl.rlim_cur = cur & RLIM_INFINITY;
+    rl.rlim_max = max & RLIM_INFINITY;
+    return setrlimit(resource, &rl);
+}
+
+""".replace('$RLIMIT_CONSTS', ''.join(rlimit_consts)))
+
+
+ffi.cdef("""
+
+#define RLIM_NLIMITS ...
+
+const struct my_rlimit_def {
+    const char *name;
+    long long value;
+} my_rlimit_consts[];
+
+struct rusage {
+    long ru_maxrss;
+    long ru_ixrss;
+    long ru_idrss;
+    long ru_isrss;
+    long ru_minflt;
+    long ru_majflt;
+    long ru_nswap;
+    long ru_inblock;
+    long ru_oublock;
+    long ru_msgsnd;
+    long ru_msgrcv;
+    long ru_nsignals;
+    long ru_nvcsw;
+    long ru_nivcsw;
+    ...;
+};
+
+static double my_utime(struct rusage *);
+static double my_stime(struct rusage *);
+void getrusage(int who, struct rusage *result);
+int my_getrlimit(int resource, long long result[2]);
+int my_setrlimit(int resource, long long cur, long long max);
+
+int wait3(int *status, int options, struct rusage *rusage);
+int wait4(int pid, int *status, int options, struct rusage *rusage);
+""")
+
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -4,151 +4,105 @@
 subprocess module on Windows.
 """
 
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The '_subprocess' module is only available on Windows")
 
 # Declare external Win32 functions
 
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
+from _pypy_winbase_cffi import ffi as _ffi
+_kernel32 = _ffi.dlopen('kernel32')
 
 GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
 
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
-
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
-
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
-
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
-
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessA
-_CreateProcess.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
 
 # Now the _subprocess module implementation
 
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
+def _WinError():
+    code, message = _ffi.getwinerror()
+    raise WindowsError(code, message)
 
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
+_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+
+class _handle(object):
+    def __init__(self, c_handle):
+        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
+        self.c_handle = c_handle
+        if int(self) != -1:
+            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
 
     def __int__(self):
-        return self.handle
+        return int(_ffi.cast("intptr_t", self.c_handle))
 
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
+    def __repr__(self):
+        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
 
     def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
+        h = int(self)
+        if h != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+        return h
 
     def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
+        if int(self) != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+            _kernel32.CloseHandle(c_handle)
 
 def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
+    handles = _ffi.new("HANDLE[2]")
 
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
+    res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size)
 
     if not res:
         raise _WinError()
 
-    return _handle(read.value), _handle(write.value)
+    return _handle(handles[0]), _handle(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
+    return _handle(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
+    # CPython: the first three arguments are expected to be integers
+    target = _ffi.new("HANDLE[1]")
 
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
+    res = _kernel32.DuplicateHandle(
+        _ffi.cast("HANDLE", source_process),
+        _ffi.cast("HANDLE", source),
+        _ffi.cast("HANDLE", target_process),
+        target, access, inherit, options)
 
     if not res:
         raise _WinError()
 
-    return _handle(target.value)
+    return _handle(target[0])
+
+def _z(input):
+    if input is None:
+        return _ffi.NULL
+    if isinstance(input, basestring):
+        return str(input)
+    raise TypeError("string/unicode/None expected, got %r" % (
+        type(input).__name__,))
 
 def CreateProcess(name, command_line, process_attr, thread_attr,
                   inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
+    si = _ffi.new("STARTUPINFO *")
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
+        # CPython: these three handles are expected to be _handle objects
         if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
+            si.hStdInput = startup_info.hStdInput.c_handle
         if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
+            si.hStdOutput = startup_info.hStdOutput.c_handle
         if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
+            si.hStdError = startup_info.hStdError.c_handle
 
-    pi = _PROCESS_INFORMATION()
+    pi = _ffi.new("PROCESS_INFORMATION *")
 
     if env is not None:
         envbuf = ""
@@ -156,47 +110,55 @@
             envbuf += "%s=%s\0" % (k, v)
         envbuf += '\0'
     else:
-        envbuf = None
+        envbuf = _ffi.NULL
 
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
+    res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL,
+                                   _ffi.NULL, inherit, flags, envbuf,
+                                   _z(start_dir), si, pi)
 
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
 
 def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
+    # CPython: the first argument is expected to be an integer.
+    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
+                                        milliseconds)
     if res < 0:
         raise _WinError()
 
     return res
 
 def GetExitCodeProcess(handle):
-    code = _c_int()
+    # CPython: the first argument is expected to be an integer.
+    code = _ffi.new("DWORD[1]")
 
-    res = _GetExitCodeProcess(int(handle), _byref(code))
+    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
 
     if not res:
         raise _WinError()
 
-    return code.value
+    return code[0]
 
 def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
+    # CPython: the first argument is expected to be an integer.
+    # The second argument is silently wrapped in a UINT.
+    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+                                     _ffi.cast("UINT", exitcode))
 
     if not res:
         raise _WinError()
 
 def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
+    stdhandle = _ffi.cast("DWORD", stdhandle)
+    res = _kernel32.GetStdHandle(stdhandle)
 
     if not res:
         return None
     else:
-        return res
+        # note: returns integer, not handle object
+        return int(_ffi.cast("intptr_t", res))
 
 STD_INPUT_HANDLE = -10
 STD_OUTPUT_HANDLE = -11
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.5.2
+Version: 1.7.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.5.2"
-__version_info__ = (1, 5, 2)
+__version__ = "1.7.0"
+__version_info__ = (1, 7, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -57,6 +57,12 @@
 # define _CFFI_UNUSED_FN  /* nothing */
 #endif
 
+#ifdef __cplusplus
+# ifndef _Bool
+#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+# endif
+#endif
+
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.5.2"
+                               "\ncompiled with cffi version: 1.7.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py
--- a/lib_pypy/cffi/_pycparser/__init__.py
+++ b/lib_pypy/cffi/_pycparser/__init__.py
@@ -10,7 +10,6 @@
 __all__ = ['c_lexer', 'c_parser', 'c_ast']
 __version__ = '2.14'
 
-from subprocess import Popen, PIPE
 from .c_parser import CParser
 
 
@@ -28,6 +27,7 @@
         When successful, returns the preprocessed file's contents.
         Errors from cpp will be printed out.
     """
+    from subprocess import Popen, PIPE
     path_list = [cpp_path]
     if isinstance(cpp_args, list):
         path_list += cpp_args
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -299,6 +299,23 @@
         """
         return self._backend.string(cdata, maxlen)
 
+    def unpack(self, cdata, length):
+        """Unpack an array of C data of the given length, 
+        returning a Python string/unicode/list.
+
+        If 'cdata' is a pointer to 'char', returns a byte string.
+        It does not stop at the first null.  This is equivalent to:
+        ffi.buffer(cdata, length)[:]
+
+        If 'cdata' is a pointer to 'wchar_t', returns a unicode string.
+        'length' is measured in wchar_t's; it is not the size in bytes.
+
+        If 'cdata' is a pointer to anything else, returns a list of
+        'length' items.  This is a faster equivalent to:
+        [cdata[i] for i in range(length)]
+        """
+        return self._backend.unpack(cdata, length)
+
     def buffer(self, cdata, size=-1):
         """Return a read-write buffer object that references the raw C data
         pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
@@ -315,8 +332,8 @@
     def from_buffer(self, python_buffer):
         """Return a <cdata 'char[]'> that points to the data of the
         given Python object, which must support the buffer interface.
-        Note that this is not meant to be used on the built-in types str,
-        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        Note that this is not meant to be used on the built-in types
+        str or unicode (you can build 'char[]' arrays explicitly)
         but only on objects containing large quantities of raw data
         in some other format, like 'array.array' or numpy arrays.
         """
@@ -380,20 +397,7 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
-        try:
-            gcp = self._backend.gcp
-        except AttributeError:
-            pass
-        else:
-            return gcp(cdata, destructor)
-        #
-        with self._lock:
-            try:
-                gc_weakrefs = self.gc_weakrefs
-            except AttributeError:
-                from .gc_weakref import GcWeakrefs
-                gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
-            return gc_weakrefs.build(cdata, destructor)
+        return self._backend.gcp(cdata, destructor)
 
     def _get_cached_btype(self, type):
         assert self._lock.acquire(False) is False
@@ -721,6 +725,26 @@
         raise ValueError("ffi.def_extern() is only available on API-mode FFI "
                          "objects")
 
+    def list_types(self):
+        """Returns the user type names known to this FFI instance.
+        This returns a tuple containing three lists of names:
+        (typedef_names, names_of_structs, names_of_unions)
+        """
+        typedefs = []
+        structs = []
+        unions = []
+        for key in self._parser._declarations:
+            if key.startswith('typedef '):
+                typedefs.append(key[8:])
+            elif key.startswith('struct '):
+                structs.append(key[7:])
+            elif key.startswith('union '):
+                unions.append(key[6:])
+        typedefs.sort()
+        structs.sort()
+        unions.sort()
+        return (typedefs, structs, unions)
+
 
 def _load_backend_lib(backend, name, flags):
     if name is None:
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -205,9 +205,7 @@
 
     def __nonzero__(self):
         return bool(self._address)
-    
-    def __bool__(self):
-        return bool(self._address)
+    __bool__ = __nonzero__
 
     @classmethod
     def _to_ctypes(cls, value):
@@ -460,6 +458,12 @@
                         return x._value
                     raise TypeError("character expected, got %s" %
                                     type(x).__name__)
+                def __nonzero__(self):
+                    return ord(self._value) != 0
+            else:
+                def __nonzero__(self):
+                    return self._value != 0
+            __bool__ = __nonzero__
 
             if kind == 'float':
                 @staticmethod
@@ -993,6 +997,31 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    def gcp(self, cdata, destructor):
+        BType = self.typeof(cdata)
+
+        if destructor is None:
+            if not (hasattr(BType, '_gcp_type') and
+                    BType._gcp_type is BType):
+                raise TypeError("Can remove destructor only on a object "
+                                "previously returned by ffi.gc()")
+            cdata._destructor = None
+            return None
+
+        try:
+            gcp_type = BType._gcp_type
+        except AttributeError:
+            class CTypesDataGcp(BType):
+                __slots__ = ['_orig', '_destructor']
+                def __del__(self):
+                    if self._destructor is not None:
+                        self._destructor(self._orig)
+            gcp_type = BType._gcp_type = CTypesDataGcp
+        new_cdata = self.cast(gcp_type, cdata)
+        new_cdata._orig = cdata
+        new_cdata._destructor = destructor
+        return new_cdata
+
     typeof = type
 
     def getcname(self, BType, replace_with):
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -35,8 +35,11 @@
                                "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == cdecl:
-                raise api.FFIError("Unsupported type: %r.  Please file a bug "
-                                   "if you think it should be." % (commontype,))
+                raise api.FFIError(
+                    "Unsupported type: %r.  Please look at "
+        "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
+                    "and file an issue if you think this type should really "
+                    "be supported." % (commontype,))
             result, quals = parser.parse_type_and_quals(cdecl)   # recursive
 
         assert isinstance(result, model.BaseTypeByIdentity)
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -29,7 +29,8 @@
 _r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b")
 _r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b")
 _r_cdecl = re.compile(r"\b__cdecl\b")
-_r_extern_python = re.compile(r'\bextern\s*"Python"\s*.')
+_r_extern_python = re.compile(r'\bextern\s*"'
+                              r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.')
 _r_star_const_space = re.compile(       # matches "* const "
     r"[*]\s*((const|volatile|restrict)\b\s*)+")
 
@@ -88,6 +89,12 @@
     #     void __cffi_extern_python_start;
     #     int foo(int);
     #     void __cffi_extern_python_stop;
+    #
+    # input: `extern "Python+C" int foo(int);`
+    # output:
+    #     void __cffi_extern_python_plus_c_start;
+    #     int foo(int);
+    #     void __cffi_extern_python_stop;
     parts = []
     while True:
         match = _r_extern_python.search(csource)
@@ -98,7 +105,10 @@
         #print ''.join(parts)+csource
         #print '=>'
         parts.append(csource[:match.start()])
-        parts.append('void __cffi_extern_python_start; ')
+        if 'C' in match.group(1):
+            parts.append('void __cffi_extern_python_plus_c_start; ')
+        else:
+            parts.append('void __cffi_extern_python_start; ')
         if csource[endpos] == '{':
             # grouping variant
             closing = csource.find('}', endpos)
@@ -302,7 +312,7 @@
                 break
         #
         try:
-            self._inside_extern_python = False
+            self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
@@ -376,8 +386,10 @@
         tp = self._get_type_pointer(tp, quals)
         if self._options.get('dllexport'):
             tag = 'dllexport_python '
-        elif self._inside_extern_python:
+        elif self._inside_extern_python == '__cffi_extern_python_start':
             tag = 'extern_python '
+        elif self._inside_extern_python == '__cffi_extern_python_plus_c_start':
+            tag = 'extern_python_plus_c '
         else:
             tag = 'function '
         self._declare(tag + decl.name, tp)
@@ -421,11 +433,9 @@
                     # hack: `extern "Python"` in the C source is replaced
                     # with "void __cffi_extern_python_start;" and
                     # "void __cffi_extern_python_stop;"
-                    self._inside_extern_python = not self._inside_extern_python
-                    assert self._inside_extern_python == (
-                        decl.name == '__cffi_extern_python_start')
+                    self._inside_extern_python = decl.name
                 else:
-                    if self._inside_extern_python:
+                    if self._inside_extern_python !='__cffi_extern_python_stop':
                         raise api.CDefError(
                             "cannot declare constants or "
                             "variables with 'extern \"Python\"'")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -814,7 +814,7 @@
             try:
                 if ftype.is_integer_type() or fbitsize >= 0:
                     # accept all integers, but complain on float or double
-                    prnt("  (void)((p->%s) << 1);  /* check that '%s.%s' is "
+                    prnt("  (void)((p->%s) | 0);  /* check that '%s.%s' is "
                          "an integer */" % (fname, cname, fname))
                     continue
                 # only accept exactly the type declared, except that '[]'
@@ -991,7 +991,7 @@
             prnt('static int %s(unsigned long long *o)' % funcname)
             prnt('{')
             prnt('  int n = (%s) <= 0;' % (name,))
-            prnt('  *o = (unsigned long long)((%s) << 0);'
+            prnt('  *o = (unsigned long long)((%s) | 0);'
                  '  /* check that %s is an integer */' % (name, name))
             if check_value is not None:
                 if check_value > 0:
@@ -1145,11 +1145,11 @@
     def _generate_cpy_extern_python_collecttype(self, tp, name):
         assert isinstance(tp, model.FunctionPtrType)
         self._do_collect_type(tp)
+    _generate_cpy_dllexport_python_collecttype = \
+      _generate_cpy_extern_python_plus_c_collecttype = \
+      _generate_cpy_extern_python_collecttype
 
-    def _generate_cpy_dllexport_python_collecttype(self, tp, name):
-        self._generate_cpy_extern_python_collecttype(tp, name)
-
-    def _generate_cpy_extern_python_decl(self, tp, name, dllexport=False):
+    def _extern_python_decl(self, tp, name, tag_and_space):
         prnt = self._prnt
         if isinstance(tp.result, model.VoidType):
             size_of_result = '0'
@@ -1184,11 +1184,7 @@
             size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % (
                 tp.result.get_c_name(''), size_of_a,
                 tp.result.get_c_name(''), size_of_a)
-        if dllexport:
-            tag = 'CFFI_DLLEXPORT'
-        else:
-            tag = 'static'
-        prnt('%s %s' % (tag, tp.result.get_c_name(name_and_arguments)))
+        prnt('%s%s' % (tag_and_space, tp.result.get_c_name(name_and_arguments)))
         prnt('{')
         prnt('  char a[%s];' % size_of_a)
         prnt('  char *p = a;')
@@ -1206,8 +1202,14 @@
         prnt()
         self._num_externpy += 1
 
+    def _generate_cpy_extern_python_decl(self, tp, name):
+        self._extern_python_decl(tp, name, 'static ')
+
     def _generate_cpy_dllexport_python_decl(self, tp, name):
-        self._generate_cpy_extern_python_decl(tp, name, dllexport=True)
+        self._extern_python_decl(tp, name, 'CFFI_DLLEXPORT ')
+
+    def _generate_cpy_extern_python_plus_c_decl(self, tp, name):
+        self._extern_python_decl(tp, name, '')
 
     def _generate_cpy_extern_python_ctx(self, tp, name):
         if self.target_is_python:
@@ -1220,8 +1222,9 @@
         self._lsts["global"].append(
             GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name))
 
-    def _generate_cpy_dllexport_python_ctx(self, tp, name):
-        self._generate_cpy_extern_python_ctx(tp, name)
+    _generate_cpy_dllexport_python_ctx = \
+      _generate_cpy_extern_python_plus_c_ctx = \
+      _generate_cpy_extern_python_ctx
 
     def _string_literal(self, s):
         def _char_repr(c):
@@ -1231,7 +1234,7 @@
             if c == '\n': return '\\n'
             return '\\%03o' % ord(c)
         lines = []
-        for line in s.splitlines(True):
+        for line in s.splitlines(True) or ['']:
             lines.append('"%s"' % ''.join([_char_repr(c) for c in line]))
         return ' \\\n'.join(lines)
 
@@ -1247,7 +1250,7 @@
 
     def _emit_bytecode_UnknownIntegerType(self, tp, index):
         s = ('_cffi_prim_int(sizeof(%s), (\n'
-             '           ((%s)-1) << 0 /* check that %s is an integer type */\n'
+             '           ((%s)-1) | 0 /* check that %s is an integer type */\n'
              '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
@@ -1319,7 +1322,9 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
-def _make_c_or_py_source(ffi, module_name, preamble, target_file):
+def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose):
+    if verbose:
+        print("generating %s" % (target_file,))
     recompiler = Recompiler(ffi, module_name,
                             target_is_python=(preamble is None))
     recompiler.collect_type_table()
@@ -1331,6 +1336,8 @@
         with open(target_file, 'r') as f1:
             if f1.read(len(output) + 1) != output:
                 raise IOError
+        if verbose:
+            print("(already up-to-date)")
         return False     # already up-to-date
     except IOError:
         tmp_file = '%s.~%d' % (target_file, os.getpid())
@@ -1343,12 +1350,14 @@
             os.rename(tmp_file, target_file)
         return True
 
-def make_c_source(ffi, module_name, preamble, target_c_file):
+def make_c_source(ffi, module_name, preamble, target_c_file, verbose=False):
     assert preamble is not None
-    return _make_c_or_py_source(ffi, module_name, preamble, target_c_file)
+    return _make_c_or_py_source(ffi, module_name, preamble, target_c_file,
+                                verbose)
 
-def make_py_source(ffi, module_name, target_py_file):
-    return _make_c_or_py_source(ffi, module_name, None, target_py_file)
+def make_py_source(ffi, module_name, target_py_file, verbose=False):
+    return _make_c_or_py_source(ffi, module_name, None, target_py_file,
+                                verbose)
 
 def _modname_to_file(outputdir, modname, extension):
     parts = modname.split('.')
@@ -1438,7 +1447,8 @@
                 target = '*'
         #
         ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds)
-        updated = make_c_source(ffi, module_name, preamble, c_file)
+        updated = make_c_source(ffi, module_name, preamble, c_file,
+                                verbose=compiler_verbose)
         if call_c_compiler:
             patchlist = []
             cwd = os.getcwd()
@@ -1458,7 +1468,8 @@
     else:
         if c_file is None:
             c_file, _ = _modname_to_file(tmpdir, module_name, '.py')
-        updated = make_py_source(ffi, module_name, c_file)
+        updated = make_py_source(ffi, module_name, c_file,
+                                 verbose=compiler_verbose)
         if call_c_compiler:
             return c_file
         else:
@@ -1484,4 +1495,7 @@
     def typeof_disabled(*args, **kwds):
         raise NotImplementedError
     ffi._typeof = typeof_disabled
+    for name in dir(ffi):
+        if not name.startswith('_') and not hasattr(module.ffi, name):
+            setattr(ffi, name, NotImplemented)
     return module.lib
diff --git a/lib_pypy/ctypes_config_cache/.empty b/lib_pypy/ctypes_config_cache/.empty
new file mode 100644
--- /dev/null
+++ b/lib_pypy/ctypes_config_cache/.empty
@@ -0,0 +1,1 @@
+dummy file to allow old buildbot configuration to run
diff --git a/lib_pypy/ctypes_config_cache/__init__.py b/lib_pypy/ctypes_config_cache/__init__.py
deleted file mode 100644
diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py
deleted file mode 100644
--- a/lib_pypy/ctypes_config_cache/dumpcache.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import sys, os
-from ctypes_configure import dumpcache
-
-def dumpcache2(basename, config):
-    size = 32 if sys.maxint <= 2**32 else 64
-    filename = '_%s_%s_.py' % (basename, size)
-    dumpcache.dumpcache(__file__, filename, config)
-    #
-    filename = os.path.join(os.path.dirname(__file__),
-                            '_%s_cache.py' % (basename,))
-    g = open(filename, 'w')
-    print >> g, '''\
-import sys
-_size = 32 if sys.maxint <= 2**32 else 64
-# XXX relative import, should be removed together with
-# XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib
-_mod = __import__("_%s_%%s_" %% (_size,),
-                  globals(), locals(), ["*"])
-globals().update(_mod.__dict__)\
-''' % (basename,)
-    g.close()
diff --git a/lib_pypy/ctypes_config_cache/locale.ctc.py b/lib_pypy/ctypes_config_cache/locale.ctc.py
deleted file mode 100644
--- a/lib_pypy/ctypes_config_cache/locale.ctc.py
+++ /dev/null
@@ -1,73 +0,0 @@
-"""
-'ctypes_configure' source for _locale.py.
-Run this to rebuild _locale_cache.py.
-"""
-
-from ctypes_configure.configure import (configure, ExternalCompilationInfo,
-    ConstantInteger, DefinedConstantInteger, SimpleType, check_eci)
-import dumpcache
-
-# ____________________________________________________________
-
-_CONSTANTS = [
-    'LC_CTYPE',
-    'LC_TIME',
-    'LC_COLLATE',
-    'LC_MONETARY',
-    'LC_MESSAGES',
-    'LC_NUMERIC',
-    'LC_ALL',
-    'CHAR_MAX',
-]
-
-class LocaleConfigure:
-    _compilation_info_ = ExternalCompilationInfo(includes=['limits.h',
-                                                           'locale.h'])
-for key in _CONSTANTS:
-    setattr(LocaleConfigure, key, DefinedConstantInteger(key))
-
-config = configure(LocaleConfigure, noerr=True)
-for key, value in config.items():
-    if value is None:
-        del config[key]
-        _CONSTANTS.remove(key)
-
-# ____________________________________________________________
-
-eci = ExternalCompilationInfo(includes=['locale.h', 'langinfo.h'])
-HAS_LANGINFO = check_eci(eci)
-
-if HAS_LANGINFO:
-    # list of all possible names
-    langinfo_names = [
-        "RADIXCHAR", "THOUSEP", "CRNCYSTR",
-        "D_T_FMT", "D_FMT", "T_FMT", "AM_STR", "PM_STR",
-        "CODESET", "T_FMT_AMPM", "ERA", "ERA_D_FMT", "ERA_D_T_FMT",
-        "ERA_T_FMT", "ALT_DIGITS", "YESEXPR", "NOEXPR", "_DATE_FMT",
-        ]
-    for i in range(1, 8):
-        langinfo_names.append("DAY_%d" % i)
-        langinfo_names.append("ABDAY_%d" % i)
-    for i in range(1, 13):
-        langinfo_names.append("MON_%d" % i)
-        langinfo_names.append("ABMON_%d" % i)
-    
-    class LanginfoConfigure:
-        _compilation_info_ = eci
-        nl_item = SimpleType('nl_item')
-    for key in langinfo_names:
-        setattr(LanginfoConfigure, key, DefinedConstantInteger(key))
-
-    langinfo_config = configure(LanginfoConfigure)
-    for key, value in langinfo_config.items():
-        if value is None:
-            del langinfo_config[key]
-            langinfo_names.remove(key)
-    config.update(langinfo_config)
-    _CONSTANTS += langinfo_names
-
-# ____________________________________________________________
-
-config['ALL_CONSTANTS'] = tuple(_CONSTANTS)
-config['HAS_LANGINFO'] = HAS_LANGINFO
-dumpcache.dumpcache2('locale', config)
diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py
deleted file mode 100755
--- a/lib_pypy/ctypes_config_cache/rebuild.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#! /usr/bin/env python
-# Run this script to rebuild all caches from the *.ctc.py files.
-
-import os, sys
-
-sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..')))
-
-import py
-
-_dirpath = os.path.dirname(__file__) or os.curdir
-
-from rpython.tool.ansi_print import AnsiLogger
-log = AnsiLogger("ctypes_config_cache")
-
-
-def rebuild_one(name):
-    filename = os.path.join(_dirpath, name)
-    d = {'__file__': filename}
-    path = sys.path[:]
-    try:
-        sys.path.insert(0, _dirpath)
-        execfile(filename, d)
-    finally:
-        sys.path[:] = path
-
-def try_rebuild():
-    size = 32 if sys.maxint <= 2**32 else 64
-    # remove the files '_*_size_.py'
-    left = {}
-    for p in os.listdir(_dirpath):
-        if p.startswith('_') and (p.endswith('_%s_.py' % size) or
-                                  p.endswith('_%s_.pyc' % size)):
-            os.unlink(os.path.join(_dirpath, p))
-        elif p.startswith('_') and (p.endswith('_.py') or
-                                    p.endswith('_.pyc')):
-            for i in range(2, len(p)-4):
-                left[p[:i]] = True
-    # remove the files '_*_cache.py' if there is no '_*_*_.py' left around
-    for p in os.listdir(_dirpath):
-        if p.startswith('_') and (p.endswith('_cache.py') or
-                                  p.endswith('_cache.pyc')):
-            if p[:-9] not in left:
-                os.unlink(os.path.join(_dirpath, p))
-    #
-    for p in os.listdir(_dirpath):
-        if p.endswith('.ctc.py'):
-            try:
-                rebuild_one(p)
-            except Exception, e:
-                log.ERROR("Running %s:\n  %s: %s" % (
-                    os.path.join(_dirpath, p),
-                    e.__class__.__name__, e))
-
-
-if __name__ == '__main__':
-    try_rebuild()
diff --git a/lib_pypy/ctypes_config_cache/resource.ctc.py b/lib_pypy/ctypes_config_cache/resource.ctc.py
deleted file mode 100644
--- a/lib_pypy/ctypes_config_cache/resource.ctc.py
+++ /dev/null
@@ -1,62 +0,0 @@
-"""
-'ctypes_configure' source for resource.py.
-Run this to rebuild _resource_cache.py.
-"""
-
-
-from ctypes import sizeof
-import dumpcache
-from ctypes_configure.configure import (configure,
-    ExternalCompilationInfo, ConstantInteger, DefinedConstantInteger,
-    SimpleType)
-
-
-_CONSTANTS = (
-    'RLIM_INFINITY',
-    'RLIM_NLIMITS',
-)
-_OPTIONAL_CONSTANTS = (
-    'RLIMIT_CPU',
-    'RLIMIT_FSIZE',
-    'RLIMIT_DATA',
-    'RLIMIT_STACK',
-    'RLIMIT_CORE',
-    'RLIMIT_RSS',
-    'RLIMIT_NPROC',
-    'RLIMIT_NOFILE',
-    'RLIMIT_OFILE',
-    'RLIMIT_MEMLOCK',
-    'RLIMIT_AS',
-    'RLIMIT_LOCKS',
-    'RLIMIT_SIGPENDING',
-    'RLIMIT_MSGQUEUE',
-    'RLIMIT_NICE',
-    'RLIMIT_RTPRIO',
-    'RLIMIT_VMEM',
-
-    'RUSAGE_BOTH',
-    'RUSAGE_SELF',
-    'RUSAGE_CHILDREN',
-)
-
-# Setup our configure
-class ResourceConfigure:
-    _compilation_info_ = ExternalCompilationInfo(includes=['sys/resource.h'])
-    rlim_t = SimpleType('rlim_t')
-for key in _CONSTANTS:
-    setattr(ResourceConfigure, key, ConstantInteger(key))
-for key in _OPTIONAL_CONSTANTS:
-    setattr(ResourceConfigure, key, DefinedConstantInteger(key))
-
-# Configure constants and types
-config = configure(ResourceConfigure)
-config['rlim_t_max'] = (1<<(sizeof(config['rlim_t']) * 8)) - 1
-optional_constants = []
-for key in _OPTIONAL_CONSTANTS:
-    if config[key] is not None:
-        optional_constants.append(key)
-    else:
-        del config[key]
-
-config['ALL_CONSTANTS'] = _CONSTANTS + tuple(optional_constants)
-dumpcache.dumpcache2('resource', config)
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -839,7 +839,7 @@
             month = self._month
         if day is None:
             day = self._day
-        return date(year, month, day)
+        return date.__new__(type(self), year, month, day)
 
     # Comparisons of date objects with other.
 
@@ -1356,7 +1356,8 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return time(hour, minute, second, microsecond, tzinfo)
+        return time.__new__(type(self),
+                            hour, minute, second, microsecond, tzinfo)
 
     def __nonzero__(self):
         if self.second or self.microsecond:
@@ -1566,8 +1567,9 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return datetime(year, month, day, hour, minute, second, microsecond,
-                        tzinfo)
+        return datetime.__new__(type(self),
+                                year, month, day, hour, minute, second,
+                                microsecond, tzinfo)
 
     def astimezone(self, tz):
         if not isinstance(tz, tzinfo):
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -7,26 +7,39 @@
 # XXX incomplete: implemented only functions needed by subprocess.py
 # PAC: 2010/08 added MS locking for Whoosh
 
-import ctypes
+# 07/2016: rewrote in CFFI
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The 'msvcrt' module is only available on Windows")
+
+import _rawffi
+from _pypy_winbase_cffi import ffi as _ffi
+_lib = _ffi.dlopen(_rawffi.get_libc().name)
+
 import errno
-from ctypes_support import standard_c_lib as _c
-from ctypes_support import get_errno
-
-try:
-    open_osfhandle = _c._open_osfhandle
-except AttributeError: # we are not on windows
-    raise ImportError
 
 try: from __pypy__ import builtinify, validate_fd
 except ImportError: builtinify = validate_fd = lambda f: f
 
 
-open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int]
-open_osfhandle.restype = ctypes.c_int
+def _ioerr():
+    e = _ffi.errno
+    raise IOError(e, errno.errorcode[e])
 
-_get_osfhandle = _c._get_osfhandle
-_get_osfhandle.argtypes = [ctypes.c_int]
-_get_osfhandle.restype = ctypes.c_int
+
+ at builtinify
+def open_osfhandle(fd, flags):
+    """"open_osfhandle(handle, flags) -> file descriptor
+
+    Create a C runtime file descriptor from the file handle handle. The
+    flags parameter should be a bitwise OR of os.O_APPEND, os.O_RDONLY,
+    and os.O_TEXT. The returned file descriptor may be used as a parameter
+    to os.fdopen() to create a file object."""
+    fd = _lib._open_osfhandle(fd, flags)
+    if fd == -1:
+        _ioerr()
+    return fd
 
 @builtinify
 def get_osfhandle(fd):
@@ -38,62 +51,74 @@
         validate_fd(fd)
     except OSError as e:
         raise IOError(*e.args)
-    return _get_osfhandle(fd)
+    result = _lib._get_osfhandle(fd)
+    if result == -1:
+        _ioerr()
+    return result
 
-setmode = _c._setmode
-setmode.argtypes = [ctypes.c_int, ctypes.c_int]
-setmode.restype = ctypes.c_int
+ at builtinify
+def setmode(fd, flags):
+    """setmode(fd, mode) -> Previous mode
+
+    Set the line-end translation mode for the file descriptor fd. To set
+    it to text mode, flags should be os.O_TEXT; for binary, it should be
+    os.O_BINARY."""
+    flags = _lib._setmode(fd, flags)
+    if flags == -1:
+        _ioerr()
+    return flags
 
 LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5)
 
-_locking = _c._locking
-_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]

From pypy.commits at gmail.com  Fri Aug  5 05:17:41 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 05 Aug 2016 02:17:41 -0700 (PDT)
Subject: [pypy-commit] pypy resource_warning: switch sockets to the
 register_finalizer approach
Message-ID: <57a459b5.8aacc20a.74e1d.e6f2@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: resource_warning
Changeset: r86026:63fff351448e
Date: 2016-08-05 11:17 +0200
http://bitbucket.org/pypy/pypy/changeset/63fff351448e/

Log:	switch sockets to the register_finalizer approach

	this is nice because it means that if resource_warnings are
	disabled, no overhead is imposed at all

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -159,16 +159,11 @@
         register_socket(space, sock)
         if self.space.sys.track_resources:
             self.w_tb = self.space.format_traceback()
+            self.register_finalizer(space)
 
-    def __del__(self):
+    def _finalize_(self):
         is_open = self.sock.fd >= 0
         if is_open and self.space.sys.track_resources:
-            self.enqueue_for_destruction(self.space, W_Socket.destructor,
-                                         '__del__ method of ')
-
-    def destructor(self):
-        assert isinstance(self, W_Socket)
-        if self.space.sys.track_resources:
             w_repr = self.space.repr(self)
             str_repr = self.space.str_w(w_repr)
             w_msg = self.space.wrap("WARNING: unclosed " + str_repr)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -455,11 +455,11 @@
           File ".*", line .*, in fn
         """, msg)
         #
-        # check with track_resources enabled in the destructor BUT with a
-        # file which was created when track_resources was disabled
+        # track_resources is enabled after the construction of the socket. in
+        # this case, the socket is not registered for finalization at all, so
+        # we don't see a message
         msg = fn(False, True)
-        assert self.regex_search("WARNING: unclosed <socket object, .*>", msg)
-        assert "Created at" not in msg
+        assert msg == ''
 
 
     def test_socket_close_error(self):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -428,4 +428,5 @@
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
 FakeObjSpace.sys.dlopenflags = 123
+FakeObjSpace.sys.track_resources = False
 FakeObjSpace.builtin = FakeModule()

From pypy.commits at gmail.com  Fri Aug  5 05:18:04 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 05 Aug 2016 02:18:04 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: jitlog encode fail arguments
 (how could I forget that :)
Message-ID: <57a459cc.262ec20a.1424d.e209@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86027:912c2e29f742
Date: 2016-08-05 11:17 +0200
http://bitbucket.org/pypy/pypy/changeset/912c2e29f742/

Log:	jitlog encode fail arguments (how could I forget that :) ppc
	misaligned_is_fine = Ture

diff --git a/rpython/rlib/rawstorage.py b/rpython/rlib/rawstorage.py
--- a/rpython/rlib/rawstorage.py
+++ b/rpython/rlib/rawstorage.py
@@ -48,7 +48,8 @@
 try:
     cpuname = detect_cpu.autodetect()
     misaligned_is_fine = cpuname.startswith('x86') or \
-                         cpuname.startswith('s390x')
+                         cpuname.startswith('s390x') or \
+                         cpuname.startswith('ppc')
     del cpuname
 except detect_cpu.ProcessorAutodetectError:
     misaligned_is_fine = False
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -509,7 +509,9 @@
         """ an operation is written as follows:
             <marker> <opid (16 bit)> \
                      <len (32 bit)> \
-                     <res_val>,<arg_0>,...,<arg_n>,<descr>
+                     <res_val>,<arg_0>,...,<arg_n> \
+                     <descr>
+                     <failarg_0>,...<failarg_n>
             The marker indicates if the last argument is
             a descr or a normal argument.
         """
@@ -518,16 +520,21 @@
         le_opnum = encode_le_16bit(op.getopnum())
         str_res = self.var_to_str(op)
         line = ','.join([str_res] + str_args)
+        failargslist = op.getfailargs()
+        failargs = ''
+        if failargslist:
+            failargs = ','.join([self.var_to_str(farg) for farg in failargslist])
+        #
         if descr:
             descr_str = descr.repr_of_descr()
             line = line + ',' + descr_str
             string = encode_str(line)
             descr_number = compute_unique_id(descr)
             le_descr_number = encode_le_addr(descr_number)
-            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number
+            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number + encode_str(failargs)
         else:
             string = encode_str(line)
-            return MARK_RESOP, le_opnum + string
+            return MARK_RESOP, le_opnum + string + encode_str(failargs)
 
 
     def write_core_dump(self, operations, i, op, ops_offset):
@@ -579,6 +586,8 @@
         return ''.join(dump)
 
     def var_to_str(self, arg):
+        if arg is None:
+            return '-'
         try:
             mv = self.memo[arg]
         except KeyError:

From pypy.commits at gmail.com  Fri Aug  5 08:18:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 05 Aug 2016 05:18:51 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: typos in draft
Message-ID: <57a4842b.031dc20a.af26a.3786@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5662:34250d9373c4
Date: 2016-08-05 14:18 +0200
http://bitbucket.org/pypy/extradoc/changeset/34250d9373c4/

Log:	typos in draft

diff --git a/blog/draft/new-jit-log.rst b/blog/draft/new-jit-log.rst
--- a/blog/draft/new-jit-log.rst
+++ b/blog/draft/new-jit-log.rst
@@ -8,14 +8,14 @@
 Both VMProf and JV share some common goals. That is the reason why they are now both packaged together.
 www.vmprof.com also got updated with various bugfixes and changes including an all new interface to JV.
 
-A advertisment: We constantly improve tooling and libraries around the Python/PyPy eco system.
+An advertisment: We constantly improve tooling and libraries around the Python/PyPy eco system.
 Here are a three examples you might also want to use in your Python projects:
 
 * VMProf - A statistical CPU profiler
 * RevDB - A reverse debugger for Python 
 * CFFI - Foreign Function Interface that avoids CPyExt
 
-A brand new JitViewer
+A "brand new" JitViewer
 ---------------------
 
 The old logging format was a hard to maintain plain text logging facility. Frequent changes often broke internal tools, most notably JV. Additionaly the logging output of a long running program took a lot of disk space.

From pypy.commits at gmail.com  Fri Aug  5 09:38:04 2016
From: pypy.commits at gmail.com (fijal)
Date: Fri, 05 Aug 2016 06:38:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: improve the test, still failing
Message-ID: <57a496bc.c19d1c0a.83f5f.0a3a@mx.google.com>

Author: fijal
Branch: 
Changeset: r86028:cb1aff2e4d19
Date: 2016-08-05 15:37 +0200
http://bitbucket.org/pypy/pypy/changeset/cb1aff2e4d19/

Log:	improve the test, still failing

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -2,7 +2,8 @@
 from rpython.rlib import jit
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rvmprof import cintf
+from rpython.rlib.rvmprof import cintf, vmprof_execute_code, register_code,\
+    register_code_object_class, _get_vmprof
 from rpython.jit.backend.x86.arch import WORD
 from rpython.jit.codewriter.policy import JitPolicy
 
@@ -14,6 +15,7 @@
 
         def helper():
             stack = cintf.vmprof_tl_stack.getraw()
+            print stack
             if stack:
                 # not during tracing
                 visited.append(stack.c_value)
@@ -22,15 +24,34 @@
 
         llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
 
-        driver = jit.JitDriver(greens=[], reds='auto')
+        driver = jit.JitDriver(greens=['code'], reds='auto')
 
-        def f(n):
+        class CodeObj(object):
+            pass
+
+        def get_code_fn(code, arg):
+            return code
+
+        def get_name(code):
+            return "foo"
+
+        register_code_object_class(CodeObj, get_name)
+
+        @vmprof_execute_code("main", get_code_fn)
+        def f(code, n):
             i = 0
             while i < n:
-                driver.jit_merge_point()
+                driver.jit_merge_point(code=code)
                 i += 1
                 llfn()
 
+        def main(n):
+            cintf.vmprof_tl_stack.setraw(null)   # make it empty
+            vmprof = _get_vmprof()
+            code = CodeObj()
+            register_code(code, get_name)
+            return f(code, n)
+
         class Hooks(jit.JitHookInterface):
             def after_compile(self, debug_info):
                 self.raw_start = debug_info.asminfo.rawstart
@@ -38,12 +59,12 @@
         hooks = Hooks()
 
         null = lltype.nullptr(cintf.VMPROFSTACK)
-        cintf.vmprof_tl_stack.setraw(null)   # make it empty
-        self.meta_interp(f, [10], policy=JitPolicy(hooks))
-        v = set(visited)
-        assert 0 in v
-        v.remove(0)
-        assert len(v) == 1
-        assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
-        assert cintf.vmprof_tl_stack.getraw() == null
+        self.meta_interp(main, [10], policy=JitPolicy(hooks))
+        print visited
+        #v = set(visited)
+        #assert 0 in v
+        #v.remove(0)
+        #assert len(v) == 1
+        #assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
+        #assert cintf.vmprof_tl_stack.getraw() == null
         # ^^^ make sure we didn't leave anything dangling
diff --git a/rpython/jit/backend/x86/test/test_rvmprof.py b/rpython/jit/backend/x86/test/test_rvmprof.py
--- a/rpython/jit/backend/x86/test/test_rvmprof.py
+++ b/rpython/jit/backend/x86/test/test_rvmprof.py
@@ -3,5 +3,5 @@
 from rpython.jit.backend.test.test_rvmprof import BaseRVMProfTest
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
 
-class TestFfiCall(Jit386Mixin, BaseRVMProfTest):
-    pass
\ No newline at end of file
+class TestRVMProfCall(Jit386Mixin, BaseRVMProfTest):
+    pass

From pypy.commits at gmail.com  Fri Aug  5 09:41:08 2016
From: pypy.commits at gmail.com (fijal)
Date: Fri, 05 Aug 2016 06:41:08 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: unskip the broken test
 on a branch
Message-ID: <57a49774.47cbc20a.13c7.4ac4@mx.google.com>

Author: fijal
Branch: improve-vmprof-testing
Changeset: r86029:a9fa0458e104
Date: 2016-08-05 15:40 +0200
http://bitbucket.org/pypy/pypy/changeset/a9fa0458e104/

Log:	unskip the broken test on a branch

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -9,8 +9,8 @@
 
 class BaseRVMProfTest(object):
     def test_one(self):
-        py.test.skip("needs thread-locals in the JIT, which is only available "
-                     "after translation")
+#        py.test.skip("needs thread-locals in the JIT, which is only available "
+#                     "after translation")
         visited = []
 
         def helper():

From pypy.commits at gmail.com  Fri Aug  5 10:04:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 05 Aug 2016 07:04:37 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: We need the vmprof stack
 updates to occur all untranslated, because the
Message-ID: <57a49cf5.c2a5c20a.adc42.5696@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86030:5af59f30c570
Date: 2016-08-05 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/5af59f30c570/

Log:	We need the vmprof stack updates to occur all untranslated, because
	the test is going to be checking that the metainterp/* updates it
	correctly, and metainterp/* is not translated here

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -37,7 +37,8 @@
 
         register_code_object_class(CodeObj, get_name)
 
-        @vmprof_execute_code("main", get_code_fn)
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
         def f(code, n):
             i = 0
             while i < n:
@@ -46,7 +47,6 @@
                 llfn()
 
         def main(n):
-            cintf.vmprof_tl_stack.setraw(null)   # make it empty
             vmprof = _get_vmprof()
             code = CodeObj()
             register_code(code, get_name)
@@ -59,6 +59,7 @@
         hooks = Hooks()
 
         null = lltype.nullptr(cintf.VMPROFSTACK)
+        cintf.vmprof_tl_stack.setraw(null)
         self.meta_interp(main, [10], policy=JitPolicy(hooks))
         print visited
         #v = set(visited)
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -4,7 +4,7 @@
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
-from rpython.rtyper.lltypesystem import rffi, llmemory
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rweaklist import RWeakListMixin
 
@@ -140,7 +140,8 @@
         if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0:
             raise VMProfError("vmprof buffers full!  disk full or too slow")
 
-def vmprof_execute_code(name, get_code_fn, result_class=None):
+def vmprof_execute_code(name, get_code_fn, result_class=None,
+                        _hack_update_stack_untranslated=False):
     """Decorator to be used on the function that interprets a code object.
 
     'name' must be a unique name.
@@ -150,6 +151,18 @@
 
     'result_class' is ignored (backward compatibility).
     """
+    if _hack_update_stack_untranslated:
+        from rpython.rtyper.annlowlevel import llhelper
+        enter_code = llhelper(lltype.Ptr(
+            lltype.FuncType([lltype.Signed], cintf.PVMPROFSTACK)),
+            cintf.enter_code)
+        leave_code = llhelper(lltype.Ptr(
+            lltype.FuncType([cintf.PVMPROFSTACK], lltype.Void)),
+            cintf.leave_code)
+    else:
+        enter_code = cintf.enter_code
+        leave_code = cintf.leave_code
+
     def decorate(func):
         try:
             _get_vmprof()
@@ -161,11 +174,11 @@
             # JIT cannot see through it.
             if not jit.we_are_jitted():
                 unique_id = get_code_fn(*args)._vmprof_unique_id
-                x = cintf.enter_code(unique_id)
+                x = enter_code(unique_id)
                 try:
                     return func(*args)
                 finally:
-                    cintf.leave_code(x)
+                    leave_code(x)
             else:
                 return func(*args)
 

From pypy.commits at gmail.com  Fri Aug  5 10:53:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 05 Aug 2016 07:53:48 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Stop using setjmp/longjmp,
 and instead kill the subprocess.  (Previously
Message-ID: <57a4a87c.6814c30a.71dcd.6fa4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86031:cf7c9337178d
Date: 2016-08-05 16:54 +0200
http://bitbucket.org/pypy/pypy/changeset/cf7c9337178d/

Log:	Stop using setjmp/longjmp, and instead kill the subprocess.
	(Previously the subprocess was in a half-zombie state.)

diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -48,6 +48,10 @@
 # if breakpoint_mode=='i': ignored, never sent
 ANSWER_BREAKPOINT = -24
 
+# sent after an Attempted to do I/O or access raw memory, as the last message
+ANSWER_ATTEMPT_IO = -25
+
+
 # print one line of a file to the console, for CMD_PRINT
 #    Message(ANSWER_LINECACHE, linenum, extra=filename)
 ANSWER_LINECACHE  = 19
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -57,6 +57,10 @@
         return a
 
 
+class RecreateSubprocess(Exception):
+    pass
+
+
 class ReplayProcess(object):
     """Represent one replaying subprocess.
 
@@ -207,6 +211,8 @@
                     pgroup.all_printed_objects_lst.append(uid)
                 sys.stdout.write('$%d = ' % nid)
                 sys.stdout.flush()
+            elif msg.cmd == ANSWER_ATTEMPT_IO:
+                raise RecreateSubprocess
             else:
                 print >> sys.stderr, "unexpected %r" % (msg,)
 
@@ -441,7 +447,8 @@
 
     def _resume(self, from_time):
         clone_me = self.paused[from_time]
-        self.active.close()
+        if self.active is not None:
+            self.active.close()
         self.active = clone_me.clone()
 
     def jump_in_time(self, target_time):
@@ -534,6 +541,12 @@
             self.active.send(Message(CMD_ATTACHID, nid, uid, int(watch_env)))
             self.active.expect_ready()
 
+    def recreate_subprocess(self):
+        # recreate a subprocess at the current time
+        time = self.get_current_time()
+        self.active = None
+        self.jump_in_time(time)
+
     def print_cmd(self, expression, nids=[]):
         """Print an expression.
         """
@@ -545,7 +558,10 @@
         self.active.tainted = True
         self.attach_printed_objects(uids, watch_env=False)
         self.active.send(Message(CMD_PRINT, extra=expression))
-        self.active.print_text_answer(pgroup=self)
+        try:
+            self.active.print_text_answer(pgroup=self)
+        except RecreateSubprocess:
+            self.recreate_subprocess()
 
     def show_backtrace(self, complete=1):
         """Show the backtrace.
@@ -553,14 +569,20 @@
         if complete:
             self.active.tainted = True
         self.active.send(Message(CMD_BACKTRACE, complete))
-        self.active.print_text_answer()
+        try:
+            self.active.print_text_answer()
+        except RecreateSubprocess:
+            self.recreate_subprocess()
 
     def show_locals(self):
         """Show the locals.
         """
         self.active.tainted = True
         self.active.send(Message(CMD_LOCALS))
-        self.active.print_text_answer()
+        try:
+            self.active.print_text_answer()
+        except RecreateSubprocess:
+            self.recreate_subprocess()
 
     def edit_breakpoints(self):
         return self.all_breakpoints
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -9,7 +9,6 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <ctype.h>
-#include <setjmp.h>
 #include <signal.h>
 #include <search.h>
 
@@ -38,7 +37,7 @@
 
 #define FID_REGULAR_MODE           'R'
 #define FID_SAVED_STATE            'S'
-#define FID_JMPBUF_PROTECTED       'J'
+#define FID_POTENTIAL_IO           'I'
 
 
 typedef struct {
@@ -566,6 +565,7 @@
 #define ANSWER_FORKED     (-22)
 #define ANSWER_AT_END     (-23)
 #define ANSWER_BREAKPOINT (-24)
+#define ANSWER_ATTEMPT_IO (-25)
 
 #define RECORD_BKPT_NUM   50
 
@@ -575,7 +575,6 @@
 static const char *rpy_rev_filename;
 static uint64_t interactive_break = 1, finalizer_break = -1, uid_break = -1;
 static uint64_t total_stop_points;
-static jmp_buf jmp_buf_cancel_execution;
 static void (*pending_after_forward)(void);
 static RPyString *empty_string;
 static uint64_t last_recorded_breakpoint_loc;
@@ -858,13 +857,12 @@
         */
         fprintf(stderr, "%s:%d: Attempted to do I/O or access raw memory\n",
                 file, line);
-        if (flag_io_disabled == FID_JMPBUF_PROTECTED) {
-            longjmp(jmp_buf_cancel_execution, 1);
-        }
-        else {
+        if (flag_io_disabled != FID_POTENTIAL_IO) {
             fprintf(stderr, "but we are not in a jmpbuf_protected section\n");
             exit(1);
         }
+        write_answer(ANSWER_ATTEMPT_IO, 0, 0, 0);
+        exit(0);
     }
 }
 
@@ -916,23 +914,24 @@
     set_revdb_breakpoints();
 }
 
-static void protect_jmpbuf(void)
+static void protect_potential_io(void)
 {
-    change_flag_io_disabled(FID_SAVED_STATE, FID_JMPBUF_PROTECTED);
+    change_flag_io_disabled(FID_SAVED_STATE, FID_POTENTIAL_IO);
     saved_exc[0] = pypy_g_ExcData.ed_exc_type;
     saved_exc[1] = pypy_g_ExcData.ed_exc_value;
     pypy_g_ExcData.ed_exc_type = NULL;
     pypy_g_ExcData.ed_exc_value = NULL;
 }
 
-static void unprotect_jmpbuf(void)
+static void unprotect_potential_io(void)
 {
-    change_flag_io_disabled(FID_JMPBUF_PROTECTED, FID_SAVED_STATE);
+    change_flag_io_disabled(FID_POTENTIAL_IO, FID_SAVED_STATE);
     if (pypy_g_ExcData.ed_exc_type != NULL) {
         fprintf(stderr, "Command crashed with %.*s\n",
                 (int)(pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.length),
                 pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.items);
-        exit(1);
+        write_answer(ANSWER_ATTEMPT_IO, 1, 0, 0);
+        exit(0);
     }
     pypy_g_ExcData.ed_exc_type = saved_exc[0];
     pypy_g_ExcData.ed_exc_value = saved_exc[1];
@@ -942,10 +941,9 @@
                                  rpy_revdb_command_t *cmd,
                                  RPyString *extra)
 {
-    protect_jmpbuf();
-    if (setjmp(jmp_buf_cancel_execution) == 0)
-        func(cmd, extra);
-    unprotect_jmpbuf();
+    protect_potential_io();
+    func(cmd, extra);
+    unprotect_potential_io();
 }
 
 static void check_at_end(uint64_t stop_points)
@@ -1267,10 +1265,9 @@
 
     save_state();
     if (rpy_revdb_commands.rp_alloc) {
-        protect_jmpbuf();
-        if (setjmp(jmp_buf_cancel_execution) == 0)
-            rpy_revdb_commands.rp_alloc(uid, new_object);
-        unprotect_jmpbuf();
+        protect_potential_io();
+        rpy_revdb_commands.rp_alloc(uid, new_object);
+        unprotect_potential_io();
     }
     uid_break = *++future_next_id;
     restore_state();
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -395,6 +395,7 @@
     def test_io_not_permitted(self):
         child = self.replay(stderr=subprocess.PIPE)
         child.send(Message(1, extra='oops'))
+        child.expect(ANSWER_ATTEMPT_IO)
         child.close()
         err = self.subproc.stderr.read()
         assert err.endswith(': Attempted to do I/O or access raw memory\n')

From pypy.commits at gmail.com  Fri Aug  5 11:19:51 2016
From: pypy.commits at gmail.com (fijal)
Date: Fri, 05 Aug 2016 08:19:51 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: work on test
Message-ID: <57a4ae97.c4431c0a.97016.2fa5@mx.google.com>

Author: fijal
Branch: improve-vmprof-testing
Changeset: r86032:31850c5d6db3
Date: 2016-08-05 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/31850c5d6db3/

Log:	work on test

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -14,22 +14,22 @@
         visited = []
 
         def helper():
+            trace = []
             stack = cintf.vmprof_tl_stack.getraw()
-            print stack
-            if stack:
-                # not during tracing
-                visited.append(stack.c_value)
-            else:
-                visited.append(0)
+            while stack:
+                trace.append((stack.c_kind, stack.c_value))
+                stack = stack.c_next
+            visited.append(trace)
 
         llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
 
         driver = jit.JitDriver(greens=['code'], reds='auto')
 
         class CodeObj(object):
-            pass
+            def __init__(self, name):
+                self.name = name
 
-        def get_code_fn(code, arg):
+        def get_code_fn(codes, code, arg):
             return code
 
         def get_name(code):
@@ -39,18 +39,21 @@
 
         @vmprof_execute_code("main", get_code_fn,
                              _hack_update_stack_untranslated=True)
-        def f(code, n):
+        def f(codes, code, n):
             i = 0
             while i < n:
                 driver.jit_merge_point(code=code)
+                if code.name == "main":
+                    f(codes, codes[1], 5)
                 i += 1
                 llfn()
 
         def main(n):
             vmprof = _get_vmprof()
-            code = CodeObj()
-            register_code(code, get_name)
-            return f(code, n)
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            return f(codes, codes[0], n)
 
         class Hooks(jit.JitHookInterface):
             def after_compile(self, debug_info):

From pypy.commits at gmail.com  Fri Aug  5 11:40:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 05 Aug 2016 08:40:41 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: More next/bnext tweaks
Message-ID: <57a4b379.81a2c20a.51926.d634@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86033:6b40ed980320
Date: 2016-08-05 17:42 +0200
http://bitbucket.org/pypy/pypy/changeset/6b40ed980320/

Log:	More next/bnext tweaks

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -512,7 +512,11 @@
         uid = 0
     else:
         uid = revdb.get_unique_id(frame)
-    revdb.send_answer(revdb.ANSWER_STACKID, uid)
+    if revdb.current_place() == -2:
+        hidden_level = 1      # hide the "<<" events from next/bnext commands
+    else:
+        hidden_level = 0
+    revdb.send_answer(revdb.ANSWER_STACKID, uid, hidden_level)
 lambda_stackid = lambda: command_stackid
 
 
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -237,47 +237,61 @@
 
     def command_next(self, argument):
         """Run forward for one step, skipping calls"""
-        stack_id = self.pgroup.get_stack_id(is_parent=False)
-        with self._stack_id_break(stack_id):
-            b = self.move_forward(1)
-        while b is not None:
-            # if we hit a regular breakpoint, stop
-            if any(b.regular_breakpoint_nums()):
-                return
-            # we hit only calls and returns inside stack_id.  If the
-            # last one of these is a "return", then we're now back inside
-            # stack_id, so stop
-            if b.nums[-1] == -2:
-                return
-            # else, the last one is a "call", so we entered another frame.
-            # Continue running until the next call/return event occurs
-            # inside stack_id
+        while True:
+            stack_id = self.pgroup.get_stack_id(is_parent=False)
             with self._stack_id_break(stack_id):
-                b = self.move_forward(self.pgroup.get_max_time() -
-                                      self.pgroup.get_current_time())
-            # and then look at that 'b' again (closes the loop)
+                b = self.move_forward(1)
+            while b is not None:
+                # if we hit a regular breakpoint, stop
+                if any(b.regular_breakpoint_nums()):
+                    return
+                # we hit only calls and returns inside stack_id.  If the
+                # last one of these is a "return", then we're now back inside
+                # stack_id, so stop
+                if b.nums[-1] == -2:
+                    break
+                # else, the last one is a "call", so we entered another frame.
+                # Continue running until the next call/return event occurs
+                # inside stack_id
+                with self._stack_id_break(stack_id):
+                    b = self.move_forward(self.pgroup.get_max_time() -
+                                          self.pgroup.get_current_time())
+                # and then look at that 'b' again (closes the loop)
+
+            # we might be at a "<<" position on the same line as before,
+            # which returns a get_hiddenpos_level() value of 1.  Continue
+            # until we reach a get_hiddenpos_level() value of 0.
+            if b is None or self.pgroup.get_hiddenpos_level() == 0:
+                break
     command_n = command_next
 
     def command_bnext(self, argument):
         """Run backward for one step, skipping calls"""
-        stack_id = self.pgroup.get_stack_id(is_parent=False)
-        with self._stack_id_break(stack_id):
-            b = self.move_backward(1)
-        while b is not None:
-            # if we hit a regular breakpoint, stop
-            if any(b.regular_breakpoint_nums()):
-                return
-            # we hit only calls and returns inside stack_id.  If the
-            # first one of these is a "call", then we're now back inside
-            # stack_id, so stop
-            if b.nums[0] == -1:
-                return
-            # else, the first one is a "return", so before, we were
-            # inside a different frame.  Continue running until the next
-            # call/return event occurs inside stack_id
+        while True:
+            stack_id = self.pgroup.get_stack_id(is_parent=False)
             with self._stack_id_break(stack_id):
-                b = self.move_backward(self.pgroup.get_current_time() - 1)
-            # and then look at that 'b' again (closes the loop)
+                b = self.move_backward(1)
+            while b is not None:
+                # if we hit a regular breakpoint, stop
+                if any(b.regular_breakpoint_nums()):
+                    return
+                # we hit only calls and returns inside stack_id.  If the
+                # first one of these is a "call", then we're now back inside
+                # stack_id, so stop
+                if b.nums[0] == -1:
+                    break
+                # else, the first one is a "return", so before, we were
+                # inside a different frame.  Continue running until the next
+                # call/return event occurs inside stack_id
+                with self._stack_id_break(stack_id):
+                    b = self.move_backward(self.pgroup.get_current_time() - 1)
+                # and then look at that 'b' again (closes the loop)
+
+            # we might be at a "<<" position on the same line as before,
+            # which returns a get_hiddenpos_level() value of 1.  Continue
+            # until we reach a get_hiddenpos_level() value of 0.
+            if self.pgroup.get_hiddenpos_level() == 0:
+                break
     command_bn = command_bnext
 
     def command_finish(self, argument):
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -587,8 +587,14 @@
     def edit_breakpoints(self):
         return self.all_breakpoints
 
+    def _stack_id(self, is_parent=0):
+        self.active.send(Message(CMD_STACKID, is_parent))
+        msg = self.active.expect(ANSWER_STACKID, Ellipsis, Ellipsis)
+        self.active.expect_ready()
+        return msg
+
     def get_stack_id(self, is_parent):
-        self.active.send(Message(CMD_STACKID, is_parent))
-        msg = self.active.expect(ANSWER_STACKID, Ellipsis)
-        self.active.expect_ready()
-        return msg.arg1
+        return self._stack_id(is_parent).arg1
+
+    def get_hiddenpos_level(self):
+        return self._stack_id().arg2

From pypy.commits at gmail.com  Fri Aug  5 11:41:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 05 Aug 2016 08:41:58 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: rewritten intro and changed title
 of blogpost
Message-ID: <57a4b3c6.e2efc20a.3d189.98bd@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5663:fde0ad51e238
Date: 2016-08-05 17:41 +0200
http://bitbucket.org/pypy/extradoc/changeset/fde0ad51e238/

Log:	rewritten intro and changed title of blogpost

diff --git a/blog/draft/new-jit-log.rst b/blog/draft/new-jit-log.rst
--- a/blog/draft/new-jit-log.rst
+++ b/blog/draft/new-jit-log.rst
@@ -1,26 +1,30 @@
-JitViewer moves to vmprof.com
+PyPy Tooling Upgrade: JitViewer and VMProf
 =======
 
-We are happy to announce that VMProf got a major update. The most significant change is the movement of JitViewer (JV)
-to VMProf.
+We are happy to announce a major JitViewer (JV) update.
 JV allows you to inspect PyPy's internal compiler representation including the generated machine code of your program.
-A useful tool to understand PyPy, learn many details of our compiler and find potential issues related to our JIT.
-Both VMProf and JV share some common goals. That is the reason why they are now both packaged together.
+A useful tool to spot issues in your program and learn PyPy's compiler details.
+
+VMProf is a statistical cpu profiler imposing very little overhead at runtime.
+
+Both VMProf and JitViewer share a common goal: Present useful information for your Python program. 
+The combination of both might reveal more information. That is the reason why they are now both packaged together.
 www.vmprof.com also got updated with various bugfixes and changes including an all new interface to JV.
 
 An advertisment: We constantly improve tooling and libraries around the Python/PyPy eco system.
-Here are a three examples you might also want to use in your Python projects:
+Here are a four examples you might also want to use in your Python projects:
 
-* VMProf - A statistical CPU profiler
-* RevDB - A reverse debugger for Python 
-* CFFI - Foreign Function Interface that avoids CPyExt
+* VMProf - A statistical CPU profiler (http://vmprof.readthedocs.io/en/latest/)
+* RevDB - A reverse debugger for Python (https://morepypy.blogspot.co.at/2016/07/reverse-debugging-for-python.html)
+* CFFI - Foreign Function Interface that avoids CPyExt (http://cffi.readthedocs.io/en/latest/)
+* JitViewer - Visualization of the log file produced by PyPy (http://vmprof.readthedocs.io/en/latest/)
 
 A "brand new" JitViewer
 ---------------------
 
-The old logging format was a hard to maintain plain text logging facility. Frequent changes often broke internal tools, most notably JV. Additionaly the logging output of a long running program took a lot of disk space.
+The old logging format was a hard to maintain plain text logging facility. Frequent changes often broke internal tools. Additionaly the logging output of a long running program took a lot of disk space.
 
-Our new binary format encodes data densly, makes use of some compression (gzip) and tries to remove repetition where possible. On top of that protocol supports versioning and can be extended easily. And *drumroll* you do not need to install JV yourself anymore! The whole system moved to vmprof.com and you can use it any time.
+Our new binary format encodes data densly, makes use of some compression (gzip) and tries to remove repetition where possible. On top of that it supports versioning and can be extended easily. And *drumroll* you do not need to install JV yourself anymore! The whole system moved to vmprof.com and you can use it any time.
 
 Sounds great. But what can you do with it? Here are two examples useful for a PyPy user:
 

From pypy.commits at gmail.com  Fri Aug  5 11:47:53 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 05 Aug 2016 08:47:53 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Allow attribute deletion on
 C-defined types
Message-ID: <57a4b529.465d1c0a.1f06f.40ae@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r86034:016781b80468
Date: 2016-08-05 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/016781b80468/

Log:	Allow attribute deletion on C-defined types

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -294,6 +294,15 @@
         assert type(obj)._some_attribute == 1
         del d["_some_attribute"]
 
+        class A(object):
+            pass
+        obj = A()
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
         d = module.get_type_dict(1)
         assert type(d) is dict
         try:
@@ -371,6 +380,21 @@
 
         api.Py_DecRef(ref)
 
+    def test_type_dict(self, space, api):
+        w_class = space.appexec([], """():
+            class A(object):
+                pass
+            return A
+            """)
+        ref = make_ref(space, w_class)
+
+        py_type = rffi.cast(PyTypeObjectPtr, ref)
+        w_dict = from_ref(space, py_type.c_tp_dict)
+        w_name = space.newunicode(u'a')
+        space.setitem(w_dict, w_name, space.wrap(1))
+        assert space.int_w(space.getattr(w_class, w_name)) == 1
+        space.delitem(w_dict, w_name)
+
     def test_multiple_inheritance(self, space, api):
         w_class = space.appexec([], """():
             class A(object):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -344,7 +344,7 @@
     def deldictvalue(self, space, key):
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        if not self.is_heaptype():
+        if not (self.is_heaptype() or self.is_cpytype()):
             raise oefmt(space.w_TypeError,
                         "can't delete attributes on type object '%N'", self)
         try:

From pypy.commits at gmail.com  Fri Aug  5 11:49:39 2016
From: pypy.commits at gmail.com (fijal)
Date: Fri, 05 Aug 2016 08:49:39 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: fight fight fight
Message-ID: <57a4b593.53b81c0a.e11d7.4049@mx.google.com>

Author: fijal
Branch: improve-vmprof-testing
Changeset: r86035:a7b374677365
Date: 2016-08-05 17:48 +0200
http://bitbucket.org/pypy/pypy/changeset/a7b374677365/

Log:	fight fight fight

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -35,6 +35,7 @@
         def get_name(code):
             return "foo"
 
+        _get_vmprof().use_weaklist = False
         register_code_object_class(CodeObj, get_name)
 
         @vmprof_execute_code("main", get_code_fn,
@@ -49,7 +50,6 @@
                 llfn()
 
         def main(n):
-            vmprof = _get_vmprof()
             codes = [CodeObj("main"), CodeObj("not main")]
             for code in codes:
                 register_code(code, get_name)
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -29,6 +29,8 @@
 
     _immutable_fields_ = ['is_enabled?']
 
+    use_weaklist = True # False for tests
+
     def __init__(self):
         "NOT_RPYTHON: use _get_vmprof()"
         self._code_classes = set()
@@ -56,7 +58,7 @@
             self._code_unique_id = uid
             if self.is_enabled:
                 self._write_code_registration(uid, full_name_func(code))
-            else:
+            elif self.use_weaklist:
                 code._vmprof_weak_list.add_handle(code)
 
     def register_code_object_class(self, CodeClass, full_name_func):
@@ -86,7 +88,8 @@
         class WeakCodeObjectList(RWeakListMixin):
             def __init__(self):
                 self.initialize()
-        CodeClass._vmprof_weak_list = WeakCodeObjectList()
+        if self.use_weaklist:
+            CodeClass._vmprof_weak_list = WeakCodeObjectList()
         #
         def gather_all_code_objs():
             all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles()

From pypy.commits at gmail.com  Fri Aug  5 11:50:53 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 05 Aug 2016 08:50:53 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Cleanup,
 delete unused MappingProxyStrategy
Message-ID: <57a4b5dd.898b1c0a.d4b9d.4247@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r86036:38670620e06c
Date: 2016-08-05 16:50 +0100
http://bitbucket.org/pypy/pypy/changeset/38670620e06c/

Log:	Cleanup, delete unused MappingProxyStrategy

diff --git a/pypy/objspace/std/classdict.py b/pypy/objspace/std/classdict.py
--- a/pypy/objspace/std/classdict.py
+++ b/pypy/objspace/std/classdict.py
@@ -2,10 +2,8 @@
 from rpython.rlib.objectmodel import iteritems_with_hash
 
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.dictmultiobject import (
-    DictStrategy, W_DictObject, create_iterator_classes)
+    DictStrategy, create_iterator_classes)
 from pypy.objspace.std.typeobject import unwrap_cell
 
 
@@ -80,7 +78,8 @@
                               for key in w_type.dict_w.iterkeys()])
 
     def values(self, w_dict):
-        return [unwrap_cell(self.space, w_value) for w_value in self.unerase(w_dict.dstorage).dict_w.itervalues()]
+        return [unwrap_cell(self.space, w_value) for w_value in
+                self.unerase(w_dict.dstorage).dict_w.itervalues()]
 
     def items(self, w_dict):
         space = self.space
@@ -100,12 +99,16 @@
 
     def getiterkeys(self, w_dict):
         return self.unerase(w_dict.dstorage).dict_w.iterkeys()
+
     def getitervalues(self, w_dict):
         return self.unerase(w_dict.dstorage).dict_w.itervalues()
+
     def getiteritems_with_hash(self, w_dict):
         return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w)
+
     def wrapkey(space, key):
         return _wrapkey(space, key)
+
     def wrapvalue(space, value):
         return unwrap_cell(space, value)
 
@@ -114,70 +117,3 @@
     return space.wrap(key.decode('utf-8'))
 
 create_iterator_classes(ClassDictStrategy)
-
-
-class MappingProxyStrategy(DictStrategy):
-    """Wraps an applevel mapping in a read-only dictionary."""
-    erase, unerase = rerased.new_erasing_pair("mappingproxy")
-    erase = staticmethod(erase)
-    unerase = staticmethod(unerase)
-
-    def getitem(self, w_dict, w_key):
-        try:
-            return self.space.getitem(self.unerase(w_dict.dstorage), w_key)
-        except OperationError as e:
-            if not e.match(self.space, self.space.w_KeyError):
-                raise
-            return None
-
-    def setitem(self, w_dict, w_key, w_value):
-        raise oefmt(self.space.w_TypeError,
-                    "'%T' object does not support item assignment", w_dict)
-
-    def delitem(self, w_dict, w_key):
-        raise oefmt(self.space.w_TypeError,
-                    "'%T' object does not support item deletion", w_dict)
-
-    def length(self, w_dict):
-        return self.space.len_w(self.unerase(w_dict.dstorage))
-
-    def getiterkeys(self, w_dict):
-        return self.space.iter(
-            self.space.call_method(self.unerase(w_dict.dstorage), "keys"))
-
-    def getitervalues(self, w_dict):
-        return self.space.iter(
-            self.space.call_method(self.unerase(w_dict.dstorage), "values"))
-
-    def getiteritems_with_hash(self, w_dict):
-        return self.space.iter(
-            self.space.call_method(self.unerase(w_dict.dstorage), "items"))
-
-    @staticmethod
-    def override_next_key(iterkeys):
-        w_keys = iterkeys.iterator
-        return iterkeys.space.next(w_keys)
-
-    @staticmethod
-    def override_next_value(itervalues):
-        w_values = itervalues.iterator
-        return itervalues.space.next(w_values)
-
-    @staticmethod
-    def override_next_item(iteritems):
-        w_items = iteritems.iterator
-        w_item = iteritems.space.next(w_items)
-        w_key, w_value = iteritems.space.unpackiterable(w_item, 2)
-        return w_key, w_value
-
-    def clear(self, w_dict):
-        raise oefmt(self.space.w_AttributeError, "clear")
-
-    def copy(self, w_dict):
-        return self.space.call_method(self.unerase(w_dict.dstorage), "copy")
-
-create_iterator_classes(
-    MappingProxyStrategy,
-    override_next_key=MappingProxyStrategy.override_next_key,
-    override_next_value=MappingProxyStrategy.override_next_value,
-    override_next_item=MappingProxyStrategy.override_next_item)

From pypy.commits at gmail.com  Fri Aug  5 11:55:16 2016
From: pypy.commits at gmail.com (fijal)
Date: Fri, 05 Aug 2016 08:55:16 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: fight fight until we win
Message-ID: <57a4b6e4.915c1c0a.c121.3f19@mx.google.com>

Author: fijal
Branch: improve-vmprof-testing
Changeset: r86037:0bcafba73720
Date: 2016-08-05 17:54 +0200
http://bitbucket.org/pypy/pypy/changeset/0bcafba73720/

Log:	fight fight until we win

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -25,6 +25,10 @@
     def __str__(self):
         return self.msg
 
+class FakeWeakCodeObjectList(object):
+    def add_handle(self, handle):
+        pass
+
 class VMProf(object):
 
     _immutable_fields_ = ['is_enabled?']
@@ -90,6 +94,8 @@
                 self.initialize()
         if self.use_weaklist:
             CodeClass._vmprof_weak_list = WeakCodeObjectList()
+        else:
+            CodeClass._vmprof_weak_list = FakeWeakCodeObjectList()
         #
         def gather_all_code_objs():
             all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles()

From pypy.commits at gmail.com  Fri Aug  5 12:06:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 05 Aug 2016 09:06:08 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57a4b970.a717c20a.155f5.9596@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r773:51149b559c70
Date: 2016-08-05 18:08 +0200
http://bitbucket.org/pypy/pypy.org/changeset/51149b559c70/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $64753 of $105000 (61.7%)
+   $64781 of $105000 (61.7%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5240 left
+    we have $5266 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Fri Aug  5 13:11:37 2016
From: pypy.commits at gmail.com (fijal)
Date: Fri, 05 Aug 2016 10:11:37 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: make sure we call llfn()
 only when in the inside code
Message-ID: <57a4c8c9.c2a5c20a.adc42.9bd8@mx.google.com>

Author: fijal
Branch: improve-vmprof-testing
Changeset: r86038:f89c232d0bfe
Date: 2016-08-05 19:10 +0200
http://bitbucket.org/pypy/pypy/changeset/f89c232d0bfe/

Log:	make sure we call llfn() only when in the inside code

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -46,8 +46,9 @@
                 driver.jit_merge_point(code=code)
                 if code.name == "main":
                     f(codes, codes[1], 5)
+                else:
+                    llfn()
                 i += 1
-                llfn()
 
         def main(n):
             codes = [CodeObj("main"), CodeObj("not main")]

From pypy.commits at gmail.com  Fri Aug  5 13:24:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 05 Aug 2016 10:24:57 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Tweaks
Message-ID: <57a4cbe9.28eac20a.8f95a.a15a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86039:9c9a38357bca
Date: 2016-08-05 19:26 +0200
http://bitbucket.org/pypy/pypy/changeset/9c9a38357bca/

Log:	Tweaks

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -253,13 +253,13 @@
 # ____________________________________________________________
 
 
-def fetch_cur_frame():
+def fetch_cur_frame(silent=False):
     ec = dbstate.space.threadlocals.get_ec()
     if ec is None:
         frame = None
     else:
         frame = ec.topframeref()
-    if frame is None:
+    if frame is None and not silent:
         revdb.send_output("No stack.\n")
     return frame
 
@@ -388,8 +388,9 @@
         indent))
     revdb.send_linecache(frame.getcode().co_filename, lineno)
 
-def display_function_part(frame, max_lines_before, max_lines_after,
-                          prompt="> "):
+def display_function_part(frame, max_lines_before, max_lines_after):
+    if frame is None:
+        return
     code = frame.getcode()
     if code.co_filename.startswith('<builtin>'):
         return
@@ -407,33 +408,33 @@
     #
     for i in range(first_lineno, final_lineno + 1):
         if i == current_lineno:
+            if revdb.current_place() == -2: # <= this is the arg to stop_point()
+                prompt = "<< "     # return
+            elif revdb.current_place() == -1:
+                prompt = "!! "     # exceptional return
+            else:
+                prompt = " > "     # plain line
             revdb.send_output(prompt)
         else:
-            revdb.send_output("  ")
+            revdb.send_output("   ")
         revdb.send_linecache(code.co_filename, i, strip=False)
     #
     if ellipsis_after:
         revdb.send_output("...\n")
 
 def command_backtrace(cmd, extra):
-    frame = fetch_cur_frame()
-    if frame is None:
-        return
+    frame = fetch_cur_frame(silent=True)
     if cmd.c_arg1 == 0:
-        revdb.send_output("%s:\n" % (
-            file_and_lineno(frame, frame.get_last_lineno()),))
-        if revdb.current_place() == -2:   # <= this is the arg to stop_point()
-            prompt = "<<"     # return
-        elif revdb.current_place() == -1:
-            prompt = "!!"     # exceptional return
-        else:
-            prompt = "> "     # plain line
-        display_function_part(frame, max_lines_before=8, max_lines_after=5,
-                              prompt=prompt)
+        if frame is not None:
+            revdb.send_output("%s:\n" % (
+                file_and_lineno(frame, frame.get_last_lineno()),))
+        display_function_part(frame, max_lines_before=8, max_lines_after=5)
     elif cmd.c_arg1 == 2:
         display_function_part(frame, max_lines_before=1000,max_lines_after=1000)
     else:
         revdb.send_output("Current call stack (most recent call last):\n")
+        if frame is None:
+            revdb.send_output("  (empty)\n")
         frames = []
         while frame is not None:
             frames.append(frame)
@@ -505,7 +506,7 @@
 
 
 def command_stackid(cmd, extra):
-    frame = fetch_cur_frame()
+    frame = fetch_cur_frame(silent=True)
     if frame is not None and cmd.c_arg1 != 0:     # parent_flag
         frame = dbstate.space.getexecutioncontext().getnextframe_nohidden(frame)
     if frame is None:

From pypy.commits at gmail.com  Fri Aug  5 13:25:10 2016
From: pypy.commits at gmail.com (fijal)
Date: Fri, 05 Aug 2016 10:25:10 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: make the first assert
 about metainterp
Message-ID: <57a4cbf6.87941c0a.9a01f.697a@mx.google.com>

Author: fijal
Branch: improve-vmprof-testing
Changeset: r86040:99215a4747c0
Date: 2016-08-05 19:24 +0200
http://bitbucket.org/pypy/pypy/changeset/99215a4747c0/

Log:	make the first assert about metainterp

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -45,7 +45,7 @@
             while i < n:
                 driver.jit_merge_point(code=code)
                 if code.name == "main":
-                    f(codes, codes[1], 5)
+                    f(codes, codes[1], 1)
                 else:
                     llfn()
                 i += 1
@@ -65,7 +65,7 @@
         null = lltype.nullptr(cintf.VMPROFSTACK)
         cintf.vmprof_tl_stack.setraw(null)
         self.meta_interp(main, [10], policy=JitPolicy(hooks))
-        print visited
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
         #v = set(visited)
         #assert 0 in v
         #v.remove(0)

From pypy.commits at gmail.com  Fri Aug  5 14:26:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 05 Aug 2016 11:26:02 -0700 (PDT)
Subject: [pypy-commit] pypy default: Another hack to avoid constant-folding
 "2 ** 12345678912"
Message-ID: <57a4da3a.c2f3c20a.a80d5.bf8e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86041:5df38f3fbcc0
Date: 2016-08-05 20:27 +0200
http://bitbucket.org/pypy/pypy/changeset/5df38f3fbcc0/

Log:	Another hack to avoid constant-folding "2 ** 12345678912"

diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -108,8 +108,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1156,3 +1156,22 @@
             counts = self.count_instructions(source)
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
+
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts

From pypy.commits at gmail.com  Fri Aug  5 14:59:21 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Fri, 05 Aug 2016 11:59:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: (plan_rich) Fix cast in
 memoryobject for rpython
Message-ID: <57a4e209.0472c20a.45b74.d088@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86042:da8ab61904e8
Date: 2016-08-05 20:58 +0200
http://bitbucket.org/pypy/pypy/changeset/da8ab61904e8/

Log:	(plan_rich) Fix cast in memoryobject for rpython

diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -147,7 +147,8 @@
 def set_sentinel(space):
     """Set a sentinel lock that will be released when the current thread
 state is finalized (after it is untied from the interpreter)."""
-    return space.wrap(Lock(space))
+    lock = allocate_lock(space)
+    return lock
 
 class W_RLock(W_Root):
     def __init__(self, space):
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -226,12 +226,12 @@
             size = rffi.sizeof(rffi.VOIDP)
         return size
     
-    def descr_cast(self, space, w_args, w_kwds):
+    def descr_cast(self, space, w_format, w_shape=None):
         # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
-        #self._check_released(space)
-        #newitemsize = self.get_native_fmtchar(w_args._val(w_args))
-        return W_MemoryView(self.buf, self.format, self.itemsize)
-        return mv
+        self._check_released(space)
+        fmt = space.str_w(w_format)
+        newitemsize = self.get_native_fmtchar(fmt)
+        return W_MemoryView(self.buf, fmt, newitemsize)
 
 
 W_MemoryView.typedef = TypeDef(

From pypy.commits at gmail.com  Fri Aug  5 15:42:37 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Fri, 05 Aug 2016 12:42:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Add tests for BUILD_MAP_UNPACK
Message-ID: <57a4ec2d.6211c20a.475ef.cdfe@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86043:0f39cfd33706
Date: 2016-08-05 21:42 +0200
http://bitbucket.org/pypy/pypy/changeset/0f39cfd33706/

Log:	Add tests for BUILD_MAP_UNPACK

diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -256,7 +256,18 @@
             return a, b, c, d
         """
         assert self.codetest(code, "f", [1, 2], {"d" : 4, "c" : 3}) == (1, 2, 3, 4)
-
+    
+    def test_build_map_unpack(self):
+        code = """
+        def f():
+            return {'x': 1, **{'y': 2}}
+        def g():
+            return {**()}
+        """
+        assert self.codetest(code, "f", []) == {'x': 1, 'y': 2}
+        res = self.codetest(code, 'g', [])
+        assert "TypeError:" in res
+        assert "'tuple' object is not a mapping" in res
 
 
 class AppTestInterpreter: 

From pypy.commits at gmail.com  Fri Aug  5 16:07:18 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Fri, 05 Aug 2016 13:07:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Add tests for
 build_set/tuple/list_unpack
Message-ID: <57a4f1f6.d42f1c0a.7bc41.8fa1@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86044:60ce08921aef
Date: 2016-08-05 22:06 +0200
http://bitbucket.org/pypy/pypy/changeset/60ce08921aef/

Log:	Add tests for build_set/tuple/list_unpack

diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -257,6 +257,24 @@
         """
         assert self.codetest(code, "f", [1, 2], {"d" : 4, "c" : 3}) == (1, 2, 3, 4)
     
+    def test_build_set_unpack(self):
+        code = """ def f():
+            return {*range(4), 4, *(5, 6, 7)}
+        """
+        assert self.codetest(code, "f", []) == {0, 1, 2, 3, 4, 5, 6, 7}
+        
+    def test_build_tuple_unpack(self):
+        code = """ def f():
+            return (*range(4), 4)
+        """
+        assert self.codetest(code, "f", []) == (0, 1, 2, 3, 4)
+    
+    def test_build_list_unpack(self):
+        code = """ def f():
+            return [*range(4), 4]
+        """
+        assert self.codetest(code, "f", []) == [0, 1, 2, 3, 4]
+    
     def test_build_map_unpack(self):
         code = """
         def f():

From pypy.commits at gmail.com  Fri Aug  5 16:13:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 05 Aug 2016 13:13:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async-translate: translation issues batch 2
Message-ID: <57a4f386.041f1c0a.342ac.959d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async-translate
Changeset: r86045:5a7b8af95db6
Date: 2016-08-05 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/5a7b8af95db6/

Log:	translation issues batch 2

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -581,12 +581,13 @@
         # position, then raise a GeneratorExit.  Otherwise, there is
         # no point.
         # If coroutine was never awaited on issue a RuntimeWarning.
-        if self.pycode is not None:
-            if self.frame is not None:
-                if self.frame.fget_f_lasti(self.frame).int_w(self.space) == -1:
-                    raise oefmt(space.w_RuntimeWarning,
-                                "coroutine '%s' was never awaited",
-                                self.pycode.co_name)
+        if self.pycode is not None and \
+           self.frame is not None and \
+           self.frame.last_instr == -1:
+            # XXX PyErr_Occured in condition?
+            raise oefmt(self.space.w_RuntimeWarning,
+                        "coroutine '%s' was never awaited",
+                        self.pycode.co_name)
         if self.frame is not None:
             block = self.frame.lastblock
             while block is not None:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -6,7 +6,7 @@
 
 from rpython.rlib import jit, rstackovf, rstring
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, always_inline
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -18,6 +18,7 @@
 from pypy.interpreter.nestedscope import Cell
 from pypy.interpreter.pycode import PyCode, BytecodeCorruption
 from pypy.tool.stdlib_opcode import bytecode_spec
+from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 
 CANNOT_CATCH_MSG = ("catching classes that don't inherit from BaseException "
                     "is not allowed in 3.x")
@@ -45,6 +46,27 @@
 
     return func_with_new_name(opimpl, "opcode_impl_for_%s" % operationname)
 
+def get_func_desc(func):
+    if self.space.type(func) is function.Function:
+        return "()"
+    elif self.space.type(func) is function.Method:
+        return "()"
+    else:
+        return " object";
+
+ at always_inline
+def list_unpack_helper(frame, itemcount):
+    space = frame.space
+    w_sum = space.newlist([], sizehint=itemcount)
+    for i in range(itemcount, 0, -1):
+        w_item = frame.peekvalue(i-1)
+        #items = frame.space.fixedview(w_item)
+        w_sum.extend(w_item)
+    while itemcount != 0:
+        frame.popvalue()
+        itemcount -= 1
+    return w_sum
+
 
 opcodedesc = bytecode_spec.opcodedesc
 HAVE_ARGUMENT = bytecode_spec.HAVE_ARGUMENT
@@ -1351,74 +1373,73 @@
             self.space.call_method(w_set, 'add', w_item)
         self.pushvalue(w_set)
 
-    def unpack_helper(self, itemcount, next_instr):
-        w_sum = []
+    def BUILD_SET_UNPACK(self, itemcount, next_instr):
+        space = self.space
+        w_sum = space.newset()
         for i in range(itemcount, 0, -1):
             w_item = self.peekvalue(i-1)
-            items = self.space.fixedview(w_item)
-            w_sum.extend(items)
+            # cannot use w_sum.update, w_item might not be a set
+            iterator = w_item.itervalues()
+            while True:
+                w_value = iterator.next_value()
+                if w_value is None:
+                    break
+                w_sum.add(w_value)
         while itemcount != 0:
             self.popvalue()
             itemcount -= 1
-        return w_sum
-
-    def BUILD_SET_UNPACK(self, itemcount, next_instr):
-        w_sum = self.unpack_helper(itemcount, next_instr)
-        self.pushvalue(self.space.newset(w_sum))
+        self.pushvalue(w_sum)
 
     def BUILD_TUPLE_UNPACK(self, itemcount, next_instr):
-        w_sum = self.unpack_helper(itemcount, next_instr)
-        self.pushvalue(self.space.newtuple(w_sum))
-        
+        space = self.space
+        w_sum_list = list_unpack_helper(self, itemcount)
+        self.pushvalue(space.newtuple(w_sum_list))
+
     def BUILD_LIST_UNPACK(self, itemcount, next_instr):
-        w_sum = self.unpack_helper(itemcount, next_instr)
-        self.pushvalue(self.space.newlist(w_sum))
-        
-    def getFuncDesc(func):
-        if self.space.type(aaa).name.decode('utf-8') == 'method':
-            return "()"
-        elif self.space.type(aaa).name.decode('utf-8') == 'function':
-            return "()"
-        else:
-            return " object";
-    
+        w_sum = list_unpack_helper(self, itemcount)
+        self.pushvalue(w_sum)
+
     def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr):
+        space = self.space
         num_maps = itemcount & 0xff
         function_location = (itemcount>>8) & 0xff
-        w_dict = self.space.newdict()
-        dict_class = w_dict.__class__
+        w_dict = space.newdict()
         for i in range(num_maps, 0, -1):
             w_item = self.peekvalue(i-1)
-            if not issubclass(w_item.__class__, dict_class):
-                raise oefmt(self.space.w_TypeError,
+            if space.lookup(w_item, '__getitem__') is None:
+                raise oefmt(space.w_TypeError,
                         "'%T' object is not a mapping", w_item)
-            num_items = w_item.length()
-            keys = w_item.w_keys()
-            for j in range(num_items):
-                if self.space.type(keys.getitem(j)).name.decode('utf-8') == 'method':
+            iterator = w_item.iterkeys()
+            while True:
+                w_key = iterator.next_key()
+                if w_key is None:
+                    break
+                if not isinstance(w_key, space.UnicodeObjectCls):
                     err_fun = self.peekvalue(num_maps + function_location-1)
-                    raise oefmt(self.space.w_TypeError,
-                        "%N%s keywords must be strings", err_fun, getFuncDesc(err_fun))
-                if self.space.is_true(self.space.contains(w_dict,keys.getitem(j))):
+                    raise oefmt(space.w_TypeError,
+                        "%N%s keywords must be strings", err_fun,
+                                                         get_func_desc(err_fun))
+                if space.is_true(space.contains(w_dict,w_key)):
                     err_fun = self.peekvalue(num_maps + function_location-1)
-                    err_arg = self.space.unicode_w(keys.getitem(j))
-                    raise oefmt(self.space.w_TypeError,
-                        "%N%s got multiple values for keyword argument %s", err_fun, getFuncDesc(err_fun), err_arg)
-            self.space.call_method(w_dict, 'update', w_item)
+                    err_arg = w_key
+                    raise oefmt(space.w_TypeError,
+                        "%N%s got multiple values for keyword argument %s",
+                          err_fun, get_func_desc(err_fun), err_arg)
+            space.call_method(w_dict, 'update', w_item)
         while num_maps != 0:
             self.popvalue()
             num_maps -= 1
         self.pushvalue(w_dict)
-        
+
     def BUILD_MAP_UNPACK(self, itemcount, next_instr):
-        w_dict = self.space.newdict()
-        dict_class = w_dict.__class__
+        space = self.space
+        w_dict = space.newdict()
         for i in range(itemcount, 0, -1):
             w_item = self.peekvalue(i-1)
-            if not issubclass(w_item.__class__, dict_class):
+            if space.lookup(w_item, '__getitem__') is None:
                 raise oefmt(self.space.w_TypeError,
                         "'%T' object is not a mapping", w_item)
-            self.space.call_method(w_dict, 'update', w_item)
+            space.call_method(w_dict, 'update', w_item)
         while itemcount != 0:
             self.popvalue()
             itemcount -= 1
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -198,7 +198,6 @@
     
     def get_native_fmtchar(self, fmt):
         from rpython.rtyper.lltypesystem import rffi
-        from sys import getsizeof
         size = -1
         if fmt[0] == '@':
             f = fmt[1]
@@ -215,7 +214,7 @@
         elif f == 'q' or f == 'Q':
             size = rffi.sizeof(rffi.LONGLONG)
         elif f == 'n' or f == 'N':
-            size = getsizeof(rffi.r_ssize_t)
+            size = rffi.sizeof(rffi.SIZE_T)
         elif f == 'f':
             size = rffi.sizeof(rffi.FLOAT)
         elif f == 'd':
@@ -225,13 +224,13 @@
         elif f == 'P':
             size = rffi.sizeof(rffi.VOIDP)
         return size
-    
-    def descr_cast(self, space, w_args, w_kwds):
+
+    def descr_cast(self, space, w_format, w_shape=None):
         # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
-        #self._check_released(space)
-        #newitemsize = self.get_native_fmtchar(w_args._val(w_args))
-        return W_MemoryView(self.buf, self.format, self.itemsize)
-        return mv
+        self._check_released(space)
+        fmt = space.str_w(w_format)
+        newitemsize = self.get_native_fmtchar(fmt)
+        return W_MemoryView(self.buf, fmt, newitemsize)
 
 
 W_MemoryView.typedef = TypeDef(

From pypy.commits at gmail.com  Sat Aug  6 03:22:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 00:22:50 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57a5904a.8f8e1c0a.694f0.22d2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r774:b523df27f1bf
Date: 2016-08-06 09:25 +0200
http://bitbucket.org/pypy/pypy.org/changeset/b523df27f1bf/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $64781 of $105000 (61.7%)
+   $64800 of $105000 (61.7%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5266 left
+    we have $5283 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -9,7 +9,7 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 38.5
+      value: 38.6
    });
   });
 </script>
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $30794 of $80000 (38.5%)
+   $30845 of $80000 (38.6%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $23459 left
+    we have $23502 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sat Aug  6 07:00:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 04:00:42 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Update the "Installing NumPy"
 section. Push numpy-via-cpyext forward.
Message-ID: <57a5c35a.c41f1c0a.c04bb.6bd6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r775:2f768f8073b0
Date: 2016-08-06 13:02 +0200
http://bitbucket.org/pypy/pypy.org/changeset/2f768f8073b0/

Log:	Update the "Installing NumPy" section. Push numpy-via-cpyext
	forward.

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -206,14 +206,38 @@
 <h1>Installing more modules</h1>
 <p>The recommended way is to install <tt class="docutils literal">pip</tt>, which is the standard package
 manager of Python.  It works like it does on CPython as explained in the
-<a class="reference external" href="http://doc.pypy.org/en/latest/install.html">installation documentation</a></p>
+<a class="reference external" href="http://doc.pypy.org/en/latest/install.html">installation documentation</a>.</p>
 </div>
 <div class="section" id="installing-numpy">
 <h1>Installing NumPy</h1>
-<p>NumPy is an exception to the rule that most packages work without
-changes.  The &ldquo;numpy&rdquo; module needs to be installed from <a class="reference external" href="https://bitbucket.org/pypy/numpy">our own
-repository</a> rather than from the official source.</p>
-<p>If you have pip:</p>
+<p><strong>There are two different versions of NumPy for PyPy.</strong></p>
+<div class="section" id="numpy-via-cpyext">
+<h2>1. NumPy via cpyext</h2>
+<p>The generally recommended way is to install the original NumPy via the
+CPython C API compatibility layer, cpyext.  Modern versions of PyPy
+support enough of the C API to make this a reasonable choice in many
+cases.  Performance-wise, the speed is mostly the same as CPython's
+NumPy (it is the same code); the exception is that interactions between
+the Python side and NumPy objects are mediated through the slower cpyext
+layer (which hurts a few benchmarks that do a lot of element-by-element
+array accesses, for example).</p>
+<p>Installation works as usual.  For example, without using a virtualenv:</p>
+<pre class="literal-block">
+$ ./pypy-xxx/bin/pypy -m ensurepip
+$ ./pypy-xxx/bin/pip install numpy
+</pre>
+<p>(See the general <a class="reference external" href="http://doc.pypy.org/en/latest/install.html">installation documentation</a> for more.)</p>
+</div>
+<div class="section" id="numpypy">
+<h2>2. NumPyPy</h2>
+<p>The &ldquo;numpy&rdquo; module can be installed from <a class="reference external" href="https://bitbucket.org/pypy/numpy">our own repository</a> rather
+than from the official source.  This version uses internally our
+built-in <tt class="docutils literal">_numpypy</tt> module.  This module is slightly incomplete.
+Also, its performance is hard to predict exactly.  For regular NumPy
+source code that handles large arrays, it is likely to be slower than
+the native NumPy with cpyext.  It is faster on the kind of code that
+contains many Python loops doing things on an element-by-element basis.</p>
+<p>Installation (see the <a class="reference external" href="http://doc.pypy.org/en/latest/install.html">installation documentation</a> for installing <tt class="docutils literal">pip</tt>):</p>
 <pre class="literal-block">
 pypy -m pip install git+https://bitbucket.org/pypy/numpy.git
 </pre>
@@ -227,10 +251,11 @@
 <pre class="literal-block">
 sudo pypy -c 'import numpy'
 </pre>
-<p>Note that NumPy support is still a work-in-progress, many things do not
-work and those that do may not be any faster than NumPy on CPython.
+<p>Note again that this version is still a work-in-progress: many things do
+not work and those that do may not be any faster than NumPy on CPython.
 For further instructions see <a class="reference external" href="https://bitbucket.org/pypy/numpy">the pypy/numpy repository</a>.</p>
 </div>
+</div>
 <div class="section" id="building-from-source">
 <span id="translate"></span><h1>Building from source</h1>
 <p>(see more build <a class="reference external" href="http://pypy.readthedocs.org/en/latest/build.html">instructions</a>)</p>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -216,7 +216,7 @@
 
 The recommended way is to install ``pip``, which is the standard package
 manager of Python.  It works like it does on CPython as explained in the 
-`installation documentation`_
+`installation documentation`_.
 
 .. _installation documentation: http://doc.pypy.org/en/latest/install.html
 
@@ -224,13 +224,43 @@
 Installing NumPy
 -------------------------------
 
-NumPy is an exception to the rule that most packages work without
-changes.  The "numpy" module needs to be installed from `our own
-repository`__ rather than from the official source.
+**There are two different versions of NumPy for PyPy.**
+
+
+1. NumPy via cpyext
++++++++++++++++++++
+
+The generally recommended way is to install the original NumPy via the
+CPython C API compatibility layer, cpyext.  Modern versions of PyPy
+support enough of the C API to make this a reasonable choice in many
+cases.  Performance-wise, the speed is mostly the same as CPython's
+NumPy (it is the same code); the exception is that interactions between
+the Python side and NumPy objects are mediated through the slower cpyext
+layer (which hurts a few benchmarks that do a lot of element-by-element
+array accesses, for example).
+
+Installation works as usual.  For example, without using a virtualenv::
+    
+    $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install numpy
+
+(See the general `installation documentation`_ for more.)
+
+
+2. NumPyPy
+++++++++++
+
+The "numpy" module can be installed from `our own repository`__ rather
+than from the official source.  This version uses internally our
+built-in ``_numpypy`` module.  This module is slightly incomplete.
+Also, its performance is hard to predict exactly.  For regular NumPy
+source code that handles large arrays, it is likely to be slower than
+the native NumPy with cpyext.  It is faster on the kind of code that
+contains many Python loops doing things on an element-by-element basis.
 
 .. __: https://bitbucket.org/pypy/numpy
 
-If you have pip::
+Installation (see the `installation documentation`_ for installing ``pip``)::
 
     pypy -m pip install git+https://bitbucket.org/pypy/numpy.git
 
@@ -244,8 +274,8 @@
 
     sudo pypy -c 'import numpy'
 
-Note that NumPy support is still a work-in-progress, many things do not
-work and those that do may not be any faster than NumPy on CPython.
+Note again that this version is still a work-in-progress: many things do
+not work and those that do may not be any faster than NumPy on CPython.
 For further instructions see `the pypy/numpy repository`__.
 
 .. __: https://bitbucket.org/pypy/numpy

From pypy.commits at gmail.com  Sat Aug  6 10:54:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 07:54:33 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: rvmprof: record the
 correct Python frames during pyjitpl
Message-ID: <57a5fa29.82ddc20a.5e0f1.056f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86046:11f391e1f1d6
Date: 2016-08-06 16:56 +0200
http://bitbucket.org/pypy/pypy/changeset/11f391e1f1d6/

Log:	rvmprof: record the correct Python frames during pyjitpl

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -452,6 +452,8 @@
             prepare = self._handle_math_sqrt_call
         elif oopspec_name.startswith('rgc.'):
             prepare = self._handle_rgc_call
+        elif oopspec_name.startswith('rvmprof.'):
+            prepare = self._handle_rvmprof_call
         elif oopspec_name.endswith('dict.lookup'):
             # also ordereddict.lookup
             prepare = self._handle_dict_lookup_call
@@ -2079,6 +2081,22 @@
         else:
             raise NotImplementedError(oopspec_name)
 
+    def _handle_rvmprof_call(self, op, oopspec_name, args):
+        if oopspec_name == 'rvmprof.enter_code':
+            leaving = 0
+        elif oopspec_name == 'rvmprof.leave_code':
+            leaving = 1
+        else:
+            raise NotImplementedError(oopspec_name)
+        c_leaving = Constant(leaving, lltype.Signed)
+        v_uniqueid = op.args[-1]
+        ops = [SpaceOperation('rvmprof_code', [c_leaving, v_uniqueid], None)]
+        if op.result.concretetype is not lltype.Void:
+            c_null = Constant(lltype.nullptr(op.result.concretetype.TO),
+                              op.result.concretetype)
+            ops.append(c_null)
+        return ops
+
     def rewrite_op_ll_read_timestamp(self, op):
         op1 = self.prepare_builtin_call(op, "ll_read_timestamp", [])
         return self.handle_residual_call(op1,
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -140,7 +140,6 @@
 
     def encoding_test(self, func, args, expected,
                       transform=False, liveness=False, cc=None, jd=None):
-        
         graphs = self.make_graphs(func, args)
         #graphs[0].show()
         if transform:
@@ -1112,6 +1111,20 @@
         assert str(e.value).startswith("A virtualizable array is passed aroun")
         assert "<Descr>" in str(e.value)
 
+    def test_rvmprof_code(self):
+        from rpython.rlib.rvmprof import cintf
+        class MyFakeCallControl(FakeCallControl):
+            def guess_call_kind(self, op):
+                return 'builtin'
+        def f(x):
+            s = cintf.enter_code(x)
+            cintf.leave_code(s, x)
+        self.encoding_test(f, [42], """
+            rvmprof_code $0, %i0
+            rvmprof_code $1, %i0
+            void_return
+        """, transform=True, cc=MyFakeCallControl())
+
 
 def check_force_cast(FROM, TO, operations, value):
     """Check that the test is correctly written..."""
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1501,6 +1501,10 @@
     def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
 
+    @arguments("i", "i")
+    def bhimpl_rvmprof_code(leaving, unique_id):
+        pass #import pdb;pdb.set_trace()
+
     # ----------
     # helpers to resume running in blackhole mode when a guard failed
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1453,6 +1453,15 @@
             metainterp.history.record(rop.VIRTUAL_REF_FINISH,
                                       [vrefbox, nullbox], None)
 
+    @arguments("int", "box")
+    def opimpl_rvmprof_code(self, leaving, box_unique_id):
+        from rpython.rlib.rvmprof import cintf
+        unique_id = box_unique_id.getint()
+        if not leaving:
+            cintf.enter_code(unique_id)
+        else:
+            cintf.leave_code_check(unique_id)
+
     # ------------------------------
 
     def setup_call(self, argboxes):
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -6,7 +6,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rlib import rthread
+from rpython.rlib import rthread, jit
 
 class VMProfPlatformUnsupported(Exception):
     pass
@@ -86,6 +86,22 @@
     ExternalCompilationInfo(includes=['vmprof_stack.h'],
                             include_dirs = [SRC]))
 
+# JIT notes:
+#
+# - When running JIT-generated assembler code, we have different custom
+#   code to build the VMPROFSTACK, so the functions below are not used.
+#
+# - The jitcode for decorated_function() in rvmprof.py still contains
+#   calls to these two oopspec functions, which are represented with
+#   the 'rvmprof_code' jitcode opcode.
+#
+# - When meta-interpreting, the 'rvmprof_code' opcode causes pyjitpl
+#   to call enter_code()/leave_code_check(), but otherwise
+#   'rvmprof_code' is ignored, i.e. doesn't produce any resop.
+#
+# - Blackhole: ...
+
+ at jit.oopspec("rvmprof.enter_code(unique_id)")
 def enter_code(unique_id):
     do_use_eci()
     s = lltype.malloc(VMPROFSTACK, flavor='raw')
@@ -95,6 +111,12 @@
     vmprof_tl_stack.setraw(s)
     return s
 
-def leave_code(s):
+ at jit.oopspec("rvmprof.leave_code(s, unique_id)")
+def leave_code(s, unique_id):
     vmprof_tl_stack.setraw(s.c_next)
     lltype.free(s, flavor='raw')
+
+def leave_code_check(unique_id):
+    s = vmprof_tl_stack.getraw()
+    assert s.c_value == unique_id
+    leave_code(s, unique_id)
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,6 +1,6 @@
 import sys, os
 from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib import jit, rposix
+from rpython.rlib import rposix
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
@@ -162,12 +162,19 @@
     """
     if _hack_update_stack_untranslated:
         from rpython.rtyper.annlowlevel import llhelper
-        enter_code = llhelper(lltype.Ptr(
+        from rpython.rlib import jit
+        enter_code_untr = llhelper(lltype.Ptr(
             lltype.FuncType([lltype.Signed], cintf.PVMPROFSTACK)),
             cintf.enter_code)
-        leave_code = llhelper(lltype.Ptr(
-            lltype.FuncType([cintf.PVMPROFSTACK], lltype.Void)),
+        leave_code_untr = llhelper(lltype.Ptr(
+            lltype.FuncType([cintf.PVMPROFSTACK, lltype.Signed], lltype.Void)),
             cintf.leave_code)
+        @jit.oopspec("rvmprof.enter_code(unique_id)")
+        def enter_code(unique_id):
+            return enter_code_untr(unique_id)
+        @jit.oopspec("rvmprof.leave_code(s)")
+        def leave_code(s, unique_id):
+            leave_code_untr(s, unique_id)
     else:
         enter_code = cintf.enter_code
         leave_code = cintf.leave_code
@@ -179,17 +186,12 @@
             return func
 
         def decorated_function(*args):
-            # If we are being JITted, we want to skip the trampoline, else the
-            # JIT cannot see through it.
-            if not jit.we_are_jitted():
-                unique_id = get_code_fn(*args)._vmprof_unique_id
-                x = enter_code(unique_id)
-                try:
-                    return func(*args)
-                finally:
-                    leave_code(x)
-            else:
+            unique_id = get_code_fn(*args)._vmprof_unique_id
+            x = enter_code(unique_id)
+            try:
                 return func(*args)
+            finally:
+                leave_code(x, unique_id)
 
         decorated_function.__name__ = func.__name__ + '_rvmprof'
         return decorated_function

From pypy.commits at gmail.com  Sat Aug  6 11:12:07 2016
From: pypy.commits at gmail.com (fijal)
Date: Sat, 06 Aug 2016 08:12:07 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: expose some other
 problems
Message-ID: <57a5fe47.031dc20a.af26a.1645@mx.google.com>

Author: fijal
Branch: improve-vmprof-testing
Changeset: r86047:8ba040124913
Date: 2016-08-06 17:11 +0200
http://bitbucket.org/pypy/pypy/changeset/8ba040124913/

Log:	expose some other problems

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -23,13 +23,13 @@
 
         llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
 
-        driver = jit.JitDriver(greens=['code'], reds='auto')
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
 
         class CodeObj(object):
             def __init__(self, name):
                 self.name = name
 
-        def get_code_fn(codes, code, arg):
+        def get_code_fn(codes, code, arg, c):
             return code
 
         def get_name(code):
@@ -40,21 +40,26 @@
 
         @vmprof_execute_code("main", get_code_fn,
                              _hack_update_stack_untranslated=True)
-        def f(codes, code, n):
+        def f(codes, code, n, c):
             i = 0
             while i < n:
-                driver.jit_merge_point(code=code)
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
                 if code.name == "main":
-                    f(codes, codes[1], 1)
+                    c = f(codes, codes[1], 1, c)
+                    driver.can_enter_jit(code=code, c=c, i=i, codes=codes, n=n)
                 else:
                     llfn()
+                    c -= 1
+                    if c < 0:
+                        llfn() # bridge
                 i += 1
+            return c
 
         def main(n):
             codes = [CodeObj("main"), CodeObj("not main")]
             for code in codes:
                 register_code(code, get_name)
-            return f(codes, codes[0], n)
+            return f(codes, codes[0], n, 8)
 
         class Hooks(jit.JitHookInterface):
             def after_compile(self, debug_info):
@@ -64,8 +69,9 @@
 
         null = lltype.nullptr(cintf.VMPROFSTACK)
         cintf.vmprof_tl_stack.setraw(null)
-        self.meta_interp(main, [10], policy=JitPolicy(hooks))
+        self.meta_interp(main, [30], policy=JitPolicy(hooks), inline=True)
         assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+        print visited
         #v = set(visited)
         #assert 0 in v
         #v.remove(0)

From pypy.commits at gmail.com  Sat Aug  6 13:29:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 10:29:40 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update the version number to 1.8
Message-ID: <57a61e84.8bc71c0a.559d7.fbba@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2732:875dcaa2b519
Date: 2016-08-06 19:32 +0200
http://bitbucket.org/cffi/cffi/changeset/875dcaa2b519/

Log:	Update the version number to 1.8

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.7.0"
+#define CFFI_VERSION  "1.8.0"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -45,9 +45,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.7'
+version = '1.8'
 # The full version, including alpha/beta/rc tags.
-release = '1.7.0'
+release = '1.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.7.0.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.0.tar.gz
 
-   - MD5: 34122a545060cee58bab88feab57006d
+   - MD5: ...
 
-   - SHA: d8033f34e17c0c51bb834b27f6e8c59fc24ae72c
+   - SHA: ...
 
-   - SHA256: 6ed5dd6afd8361f34819c68aaebf9e8fc12b5a5893f91f50c9e50c8886bb60df
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.7.0',
+        version='1.8.0',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Sat Aug  6 13:33:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 10:33:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: Update to cffi 1.8.0
Message-ID: <57a61f73.17a61c0a.a160.fc13@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86048:fa7bd4dae2a8
Date: 2016-08-06 19:35 +0200
http://bitbucket.org/pypy/pypy/changeset/fa7bd4dae2a8/

Log:	Update to cffi 1.8.0

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -196,20 +196,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.7.0"
+VERSION = "1.8.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -130,7 +130,7 @@
                 cls.module = str(udir.join('testownlib.dll'))
         else:
             subprocess.check_call(
-                'gcc testownlib.c -shared -fPIC -o testownlib.so',
+                'cc testownlib.c -shared -fPIC -o testownlib.so',
                 cwd=str(udir), shell=True)
             cls.module = str(udir.join('testownlib.so'))
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -852,9 +852,12 @@
     assert str(e2.value) == "foo0() takes no arguments (2 given)"
     assert str(e3.value) == "foo1() takes exactly one argument (0 given)"
     assert str(e4.value) == "foo1() takes exactly one argument (2 given)"
-    assert str(e5.value) == "foo2() takes exactly 2 arguments (0 given)"
-    assert str(e6.value) == "foo2() takes exactly 2 arguments (1 given)"
-    assert str(e7.value) == "foo2() takes exactly 2 arguments (3 given)"
+    assert str(e5.value) in ["foo2 expected 2 arguments, got 0",
+                             "foo2() takes exactly 2 arguments (0 given)"]
+    assert str(e6.value) in ["foo2 expected 2 arguments, got 1",
+                             "foo2() takes exactly 2 arguments (1 given)"]
+    assert str(e7.value) in ["foo2 expected 2 arguments, got 3",
+                             "foo2() takes exactly 2 arguments (3 given)"]
 
 def test_address_of_function():
     ffi = FFI()
@@ -1916,3 +1919,35 @@
     ffi.cdef("bool f(void);")
     lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
     assert lib.f() == 1
+
+def test_struct_field_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[2]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+
+def test_function_arg_opaque():
+    py.test.skip("can currently declare a function with an opaque struct "
+                 "as argument, but AFAICT it's impossible to call it later")
+
+def test_function_returns_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a foo(int);")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_function_returns_opaque", "?")
+    assert str(e.value) == ("function foo: 'struct a' is used as result type,"
+                            " but is opaque")

From pypy.commits at gmail.com  Sat Aug  6 14:17:47 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 06 Aug 2016 11:17:47 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: suggest some tweaks,
 feel free to revert
Message-ID: <57a629cb.8cc51c0a.2ceb1.0ca2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r5664:d5eaf9f670d2
Date: 2016-08-06 21:17 +0300
http://bitbucket.org/pypy/extradoc/changeset/d5eaf9f670d2/

Log:	suggest some tweaks, feel free to revert

diff --git a/blog/draft/new-jit-log.rst b/blog/draft/new-jit-log.rst
--- a/blog/draft/new-jit-log.rst
+++ b/blog/draft/new-jit-log.rst
@@ -2,38 +2,51 @@
 =======
 
 We are happy to announce a major JitViewer (JV) update.
-JV allows you to inspect PyPy's internal compiler representation including the generated machine code of your program.
-A useful tool to spot issues in your program and learn PyPy's compiler details.
+JV allows you to inspect RPython's internal compiler representation (the language in which PyPy is implemented)
+including the generated machine code of your program.
+It can graphically show you details of the RPython compiled code and helps you pinpoint issues in your code.
 
-VMProf is a statistical cpu profiler imposing very little overhead at runtime.
+VMProf is a statistical CPU profiler for python imposing very little overhead at runtime.
 
-Both VMProf and JitViewer share a common goal: Present useful information for your Python program. 
-The combination of both might reveal more information. That is the reason why they are now both packaged together.
-www.vmprof.com also got updated with various bugfixes and changes including an all new interface to JV.
+Both VMProf and JitViewer share a common goal: Present useful information for your python program. 
+The combination of both can reveal more information than either alone. 
+That is the reason why they are now both packaged together.
+We also updated www.vmprof.com with various bug fixes and changes including an all new interface to JV.
 
-An advertisment: We constantly improve tooling and libraries around the Python/PyPy eco system.
-Here are a four examples you might also want to use in your Python projects:
+This work was done with the goal of improving tooling and libraries around the Python/PyPy/RPython ecosystem.
+Some of the tools we have developed:
+
+* CFFI - Foreign Function Interface that avoids CPyExt (http://cffi.readthedocs.io/en/latest/)
+* RevDB - A reverse debugger for python (https://morepypy.blogspot.co.at/2016/07/reverse-debugging-for-python.html)
+
+and of course the tools we discuss here:
 
 * VMProf - A statistical CPU profiler (http://vmprof.readthedocs.io/en/latest/)
-* RevDB - A reverse debugger for Python (https://morepypy.blogspot.co.at/2016/07/reverse-debugging-for-python.html)
-* CFFI - Foreign Function Interface that avoids CPyExt (http://cffi.readthedocs.io/en/latest/)
-* JitViewer - Visualization of the log file produced by PyPy (http://vmprof.readthedocs.io/en/latest/)
+* JitViewer - Visualization of the log file produced by RPython (http://vmprof.readthedocs.io/en/latest/)
 
 A "brand new" JitViewer
 ---------------------
 
-The old logging format was a hard to maintain plain text logging facility. Frequent changes often broke internal tools. Additionaly the logging output of a long running program took a lot of disk space.
+JitViewer has two pieces: you create a log file when running your program, and then use a graphic tool to view what happened.
 
-Our new binary format encodes data densly, makes use of some compression (gzip) and tries to remove repetition where possible. On top of that it supports versioning and can be extended easily. And *drumroll* you do not need to install JV yourself anymore! The whole system moved to vmprof.com and you can use it any time.
+The old logging format was a hard-to-maintain, plain-text-logging facility. Frequent changes often broke internal tools. 
+Additionally, the logging output of a long running program required a lot of disk space.
 
-Sounds great. But what can you do with it? Here are two examples useful for a PyPy user:
+Our new binary format encodes data densely, makes use of some compression (gzip), and tries to remove repetition where possible. 
+It also supports versioning for future proofing and can be extended easily. 
+
+And *drumroll* you no longer need to install a tool to view the log yourself
+anymore! The whole system moved to vmprof.com and you can use it any time.
+
+Sounds great. But what can you do with it? Here are two examples for a PyPy user:
 
 PyPy crashed? Did you discover a bug?
 -------------------
 
-For some hard to find bugs it is often necessary to look at the compiled code. The old procedure often required to upload a plain text file which was hard to parse and to look through. 
+For some hard to find bugs it is often necessary to look at the compiled code. The old
+procedure often required you to upload a plain text file which was hard to parse and to look through. 
 
-A new way to share a crash report is to install the ``vmprof`` module from PyPi and execute either of the two commands:
+A better way to share a crash report is to install the ``vmprof`` module from PyPi and execute either of the two commands:
 
 ```
 # this program does not crash, but has some weird behaviour
@@ -48,12 +61,12 @@
 PyPy Jitlog: http://vmprof.com/#/<hash>/traces
 ```
 
-Providing the link in the bug report enables PyPy developers browse and identify potential issues.
+Providing the link in the bug report allows PyPy developers to browse and identify potential issues.
 
 Speed issues
 ------------
 
-VMProf is a great tool to find out hot spots that consume a lot of time in your program. As soon as you have idenified code that runs slow, you can switch to jitlog and maybe pin point certain aspects that do not behave as expected. You will find not only the overview, but are also able to browse the generated code. If you cannot make sense of that all you can just share the link with us and we can have a look at too.
+VMProf is a great tool to find out hot spots that consume a lot of time in your program. As soon as you have identified code that runs slowly, you can switch to jitlog and maybe pinpoint certain aspects that do not behave as expected. You will find an overview, and are able to browse the generated code. If you cannot make sense of all that, you can just share the link with us and we can have a look too.
 
 Future direction
 ----------------
@@ -62,11 +75,11 @@
 
 Here are a few ideas what might come in the next few releases:
 
-* Combination of CPU profiles and the JITLOG (Sadly did not make it into the current release)
+* Combination of CPU profiles and the JITLOG (sadly did not make it into the current release).
 
 * Extend vmprof.com to be able to query vmprof/jitlog. An example query for vmprof: 'methods.callsites() > 5' and for the jitlog would be 'traces.contains('call_assembler').hasbridge('*my_func_name*')'.
 
-* Extend the jitlog to capture the information of the optimization stage
+* Extend the jitlog to capture the information of the optimization stage.
 
 Richard Plangger (plan_rich) and the PyPy team
 

From pypy.commits at gmail.com  Sat Aug  6 14:24:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 11:24:54 -0700 (PDT)
Subject: [pypy-commit] cffi default: Seems that as a #define,
 this hack breaks if a future header does
Message-ID: <57a62b76.c70a1c0a.ea799.0509@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2733:6ddf5a9d1409
Date: 2016-08-06 20:27 +0200
http://bitbucket.org/cffi/cffi/changeset/6ddf5a9d1409/

Log:	Seems that as a #define, this hack breaks if a future header does
	"typedef bool _Bool;". With a typedef, though, we get two identical
	typedefs, which at least g++ is perfectly happy with

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -59,7 +59,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 

From pypy.commits at gmail.com  Sat Aug  6 14:41:00 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sat, 06 Aug 2016 11:41:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Fix test for build_set_unpack,
 add test for build_map_unpack_with_call, fix getFuncDesc
Message-ID: <57a62f3c.4bc41c0a.32732.0e98@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86049:7f49d892fb4d
Date: 2016-08-06 20:40 +0200
http://bitbucket.org/pypy/pypy/changeset/7f49d892fb4d/

Log:	Fix test for build_set_unpack, add test for
	build_map_unpack_with_call, fix getFuncDesc

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1374,10 +1374,10 @@
         w_sum = self.unpack_helper(itemcount, next_instr)
         self.pushvalue(self.space.newlist(w_sum))
         
-    def getFuncDesc(func):
-        if self.space.type(aaa).name.decode('utf-8') == 'method':
+    def getFuncDesc(self, func):
+        if self.space.type(func).name.decode('utf-8') == 'method':
             return "()"
-        elif self.space.type(aaa).name.decode('utf-8') == 'function':
+        elif self.space.type(func).name.decode('utf-8') == 'function':
             return "()"
         else:
             return " object";
@@ -1403,7 +1403,7 @@
                     err_fun = self.peekvalue(num_maps + function_location-1)
                     err_arg = self.space.unicode_w(keys.getitem(j))
                     raise oefmt(self.space.w_TypeError,
-                        "%N%s got multiple values for keyword argument %s", err_fun, getFuncDesc(err_fun), err_arg)
+                        "%N%s got multiple values for keyword argument '%s'", err_fun, self.getFuncDesc(err_fun), err_arg)
             self.space.call_method(w_dict, 'update', w_item)
         while num_maps != 0:
             self.popvalue()
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -261,7 +261,10 @@
         code = """ def f():
             return {*range(4), 4, *(5, 6, 7)}
         """
-        assert self.codetest(code, "f", []) == {0, 1, 2, 3, 4, 5, 6, 7}
+        space = self.space
+        res = self.codetest(code, "f", [])
+        l_res = space.call_function(space.w_list, res)
+        assert space.unwrap(l_res) == [0, 1, 2, 3, 4, 5, 6, 7]
         
     def test_build_tuple_unpack(self):
         code = """ def f():
@@ -286,6 +289,30 @@
         res = self.codetest(code, 'g', [])
         assert "TypeError:" in res
         assert "'tuple' object is not a mapping" in res
+    
+    def test_build_map_unpack_with_call(self):
+        code = """
+        def f(a,b,c,d):
+            return a+b,c+d
+        def g1():
+            return f(**{'a': 1, 'c': 3}, **{'b': 2, 'd': 4})
+        def g2():
+            return f(**{'a': 1, 'c': 3}, **[])
+        def g3():
+            return f(**{'a': 1, 'c': 3}, **{1: 3})
+        def g4():
+            return f(**{'a': 1, 'c': 3}, **{'a': 2})
+        """
+        assert self.codetest(code, "g1", []) == (3, 7)
+        resg2 = self.codetest(code, 'g2', [])
+        assert "TypeError:" in resg2
+        assert "'list' object is not a mapping" in resg2
+        resg3 = self.codetest(code, 'g3', [])
+        assert "TypeError:" in resg3
+        assert "keywords must be strings" in resg3
+        resg4 = self.codetest(code, 'g4', [])
+        assert "TypeError:" in resg4
+        assert "f() got multiple values for keyword argument 'a'" in resg4
 
 
 class AppTestInterpreter: 

From pypy.commits at gmail.com  Sat Aug  6 14:49:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 11:49:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: Import cffi/6ddf5a9d1409
Message-ID: <57a63151.45c8c20a.3d264.b421@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86050:67a7286bfdd2
Date: 2016-08-06 20:49 +0200
http://bitbucket.org/pypy/pypy/changeset/67a7286bfdd2/

Log:	Import cffi/6ddf5a9d1409

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -59,7 +59,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 

From pypy.commits at gmail.com  Sat Aug  6 15:02:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 12:02:43 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update docs
Message-ID: <57a63453.4317c20a.5ff9b.5d62@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2734:afc6e9c21056
Date: 2016-08-06 21:05 +0200
http://bitbucket.org/cffi/cffi/changeset/afc6e9c21056/

Log:	Update docs

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -837,8 +837,8 @@
 .. __: distutils-setuptools_
 
 The following example should work both with old (pre-1.0) and new
-versions of CFFI---supporting both is important to run on PyPy,
-because CFFI 1.0 does not work in PyPy < 2.6:
+versions of CFFI---supporting both is important to run on old
+versions of PyPy (CFFI 1.0 does not work in PyPy < 2.6):
 
 .. code-block:: python
 
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -627,12 +627,12 @@
    *`` argument might be passed as ``[[x, y]]`` or ``[{'x': 5, 'y':
    10}]``.
 
-   As an optimization, the CPython version of CFFI assumes that a
+   As an optimization, CFFI assumes that a
    function with a ``char *`` argument to which you pass a Python
    string will not actually modify the array of characters passed in,
    and so passes directly a pointer inside the Python string object.
-   (PyPy might in the future do the same, but it is harder because
-   strings are not naturally zero-terminated in PyPy.)
+   (On PyPy, this optimization is only available since PyPy 5.4
+   with CFFI 1.8.)
 
 `(**)` C function calls are done with the GIL released.
 
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -7,7 +7,14 @@
 ====
 
 * Removed the restriction that ``ffi.from_buffer()`` cannot be used on
-  byte strings (PyPy was improved and can now support that case).
+  byte strings.  Now you can get a ``char *`` out of a byte string,
+  which is valid as long as the string object is kept alive.  (But
+  don't use it to *modify* the string object!  If you need this, use
+  ``bytearray`` or other official techniques.)
+
+* PyPy 5.4 can now pass a byte string directly to a ``char *``
+  argument (in older versions, a copy would be made).  This used to be
+  a CPython-only optimization.
 
 
 v1.7

From pypy.commits at gmail.com  Sat Aug  6 16:13:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 13:13:57 -0700 (PDT)
Subject: [pypy-commit] cffi default: Avoid a combination where we end up
 with two incompatible "typedef xxx
Message-ID: <57a64505.03121c0a.f3fb7.2b71@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2735:d9c892b5a64b
Date: 2016-08-06 22:16 +0200
http://bitbucket.org/cffi/cffi/changeset/d9c892b5a64b/

Log:	Avoid a combination where we end up with two incompatible "typedef
	xxx _Bool;"

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1919,6 +1919,18 @@
     lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
     assert lib.f() == 1
 
+def test_bool_in_cpp_2():
+    ffi = FFI()
+    ffi.cdef('int add(int a, int b);')
+    lib = verify(ffi, "test_bool_bug_cpp", '''
+        typedef bool _Bool;  /* there is a Windows header with this line */
+        int add(int a, int b)
+        {
+            return a + b;
+        }''', source_extension='.cpp')
+    c = lib.add(2, 3)
+    assert c == 5
+
 def test_struct_field_opaque():
     ffi = FFI()
     ffi.cdef("struct a { struct b b; };")

From pypy.commits at gmail.com  Sat Aug  6 16:15:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 06 Aug 2016 13:15:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: update cffi/d9c892b5a64b
Message-ID: <57a64551.919a1c0a.d74cb.2be9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86051:68c75f95713b
Date: 2016-08-06 22:17 +0200
http://bitbucket.org/pypy/pypy/changeset/68c75f95713b/

Log:	update cffi/d9c892b5a64b

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1920,6 +1920,18 @@
     lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
     assert lib.f() == 1
 
+def test_bool_in_cpp_2():
+    ffi = FFI()
+    ffi.cdef('int add(int a, int b);')
+    lib = verify(ffi, "test_bool_bug_cpp", '''
+        typedef bool _Bool;  /* there is a Windows header with this line */
+        int add(int a, int b)
+        {
+            return a + b;
+        }''', source_extension='.cpp')
+    c = lib.add(2, 3)
+    assert c == 5
+
 def test_struct_field_opaque():
     ffi = FFI()
     ffi.cdef("struct a { struct b b; };")

From pypy.commits at gmail.com  Sun Aug  7 03:30:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 00:30:39 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: in-progress
Message-ID: <57a6e39f.a717c20a.155f5.0a4c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86052:271af761ce0b
Date: 2016-08-07 09:32 +0200
http://bitbucket.org/pypy/pypy/changeset/271af761ce0b/

Log:	in-progress

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -7,7 +7,73 @@
 from rpython.jit.backend.x86.arch import WORD
 from rpython.jit.codewriter.policy import JitPolicy
 
+
 class BaseRVMProfTest(object):
+
+    def setup_method(self, meth):
+        visited = []
+
+        def helper():
+            trace = []
+            stack = cintf.vmprof_tl_stack.getraw()
+            while stack:
+                trace.append((stack.c_kind, stack.c_value))
+                stack = stack.c_next
+            visited.append(trace)
+
+        llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
+
+        class CodeObj(object):
+            def __init__(self, name):
+                self.name = name
+
+        def get_code_fn(codes, code, arg, c):
+            return code
+
+        def get_name(code):
+            return "foo"
+
+        _get_vmprof().use_weaklist = False
+        register_code_object_class(CodeObj, get_name)
+
+        self.misc = visited, llfn, CodeObj, get_code_fn, get_name
+
+
+    def teardown_method(self, meth):
+        del _get_vmprof().use_weaklist
+
+
+    def test_simple(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
+            i = 0
+            while i < n:
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if code.name == "main":
+                    c = f(codes, codes[1], 1, c)
+                    driver.can_enter_jit(code=code, c=c, i=i, codes=codes, n=n)
+                else:
+                    llfn()
+                    c -= 1
+                i += 1
+            return c
+
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            return f(codes, codes[0], n, 8)
+
+        null = lltype.nullptr(cintf.VMPROFSTACK)
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+
+
     def test_one(self):
 #        py.test.skip("needs thread-locals in the JIT, which is only available "
 #                     "after translation")
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -2082,20 +2082,10 @@
             raise NotImplementedError(oopspec_name)
 
     def _handle_rvmprof_call(self, op, oopspec_name, args):
-        if oopspec_name == 'rvmprof.enter_code':
-            leaving = 0
-        elif oopspec_name == 'rvmprof.leave_code':
-            leaving = 1
-        else:
+        if oopspec_name != 'rvmprof.code':
             raise NotImplementedError(oopspec_name)
-        c_leaving = Constant(leaving, lltype.Signed)
-        v_uniqueid = op.args[-1]
-        ops = [SpaceOperation('rvmprof_code', [c_leaving, v_uniqueid], None)]
-        if op.result.concretetype is not lltype.Void:
-            c_null = Constant(lltype.nullptr(op.result.concretetype.TO),
-                              op.result.concretetype)
-            ops.append(c_null)
-        return ops
+        c_leaving, v_uniqueid = args
+        return SpaceOperation('rvmprof_code', [c_leaving, v_uniqueid], None)
 
     def rewrite_op_ll_read_timestamp(self, op):
         op1 = self.prepare_builtin_call(op, "ll_read_timestamp", [])
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -14,7 +14,7 @@
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from rpython.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rlib import jit
+from rpython.rlib import jit, debug
 
 
 class FakeRegAlloc:
@@ -1115,14 +1115,26 @@
         from rpython.rlib.rvmprof import cintf
         class MyFakeCallControl(FakeCallControl):
             def guess_call_kind(self, op):
-                return 'builtin'
+                if '_code' in repr(op):
+                    return 'builtin'
+                return 'residual'
+        class X:
+            pass
+        def g():
+            debug.debug_print("foo")
+            return X()
+        g._dont_inline_ = True
         def f(x):
-            s = cintf.enter_code(x)
-            cintf.leave_code(s, x)
+            cintf.jit_rvmprof_code(0, x)
+            res = g()
+            cintf.jit_rvmprof_code(1, x)
+            return res
         self.encoding_test(f, [42], """
             rvmprof_code $0, %i0
+            residual_call_r_r $<* fn g>, R[], <Descr> -> %r0
+            -live-
             rvmprof_code $1, %i0
-            void_return
+            ref_return %r0
         """, transform=True, cc=MyFakeCallControl())
 
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -74,6 +74,8 @@
         self.parent_snapshot = None
         # counter for unrolling inlined loops
         self.unroll_iterations = 1
+        # rvmprof
+        self.rvmprof_unique_id = -1
 
     @specialize.arg(3)
     def copy_constants(self, registers, constants, ConstClass):
@@ -1456,11 +1458,7 @@
     @arguments("int", "box")
     def opimpl_rvmprof_code(self, leaving, box_unique_id):
         from rpython.rlib.rvmprof import cintf
-        unique_id = box_unique_id.getint()
-        if not leaving:
-            cintf.enter_code(unique_id)
-        else:
-            cintf.leave_code_check(unique_id)
+        cintf.jit_rvmprof_code(leaving, box_unique_id.getint())
 
     # ------------------------------
 
@@ -1813,6 +1811,7 @@
             opimpl = _get_opimpl_method(name, argcodes)
             self.opcode_implementations[value] = opimpl
         self.op_catch_exception = insns.get('catch_exception/L', -1)
+        self.op_rvmprof_code = insns.get('rvmprof_code/ii', -1)
 
     def setup_descrs(self, descrs):
         self.opcode_descrs = descrs
@@ -2080,6 +2079,10 @@
                     target = ord(code[position+1]) | (ord(code[position+2])<<8)
                     frame.pc = target
                     raise ChangeFrame
+                if opcode == self.staticdata.op_rvmprof_code:
+                    # do the 'leave_code' for rvmprof, but then continue
+                    # popping frames
+                    import pdb;pdb.set_trace()
             self.popframe()
         try:
             self.compile_exit_frame_with_exception(self.last_exc_box)
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -86,22 +86,6 @@
     ExternalCompilationInfo(includes=['vmprof_stack.h'],
                             include_dirs = [SRC]))
 
-# JIT notes:
-#
-# - When running JIT-generated assembler code, we have different custom
-#   code to build the VMPROFSTACK, so the functions below are not used.
-#
-# - The jitcode for decorated_function() in rvmprof.py still contains
-#   calls to these two oopspec functions, which are represented with
-#   the 'rvmprof_code' jitcode opcode.
-#
-# - When meta-interpreting, the 'rvmprof_code' opcode causes pyjitpl
-#   to call enter_code()/leave_code_check(), but otherwise
-#   'rvmprof_code' is ignored, i.e. doesn't produce any resop.
-#
-# - Blackhole: ...
-
- at jit.oopspec("rvmprof.enter_code(unique_id)")
 def enter_code(unique_id):
     do_use_eci()
     s = lltype.malloc(VMPROFSTACK, flavor='raw')
@@ -111,12 +95,52 @@
     vmprof_tl_stack.setraw(s)
     return s
 
- at jit.oopspec("rvmprof.leave_code(s, unique_id)")
-def leave_code(s, unique_id):
+def leave_code(s):
     vmprof_tl_stack.setraw(s.c_next)
     lltype.free(s, flavor='raw')
 
-def leave_code_check(unique_id):
-    s = vmprof_tl_stack.getraw()
-    assert s.c_value == unique_id
-    leave_code(s, unique_id)
+#
+# JIT notes:
+#
+# - When running JIT-generated assembler code, we have different custom
+#   code to build the VMPROFSTACK, so the functions above are not used.
+#   (It uses kind == VMPROF_JITTED_TAG and the VMPROFSTACK is allocated
+#   in the C stack.)
+#
+# - The jitcode for decorated_jitted_function() in rvmprof.py, if
+#   we_are_jitted() calls the oopspec'ed function jit_rvmprof_code(),
+#   which turns into a simple jitcode opcode.  The jitcode has a
+#   simple structure:
+#
+#        rvmprof_code(0, unique_id)
+#        res = inline_call FUNC
+#        rvmprof_code(1, unique_id)
+#
+#   with no catch_exception logic for a "finally:" block.  Instead the
+#   blackhole interp looks for this simple pattern.  This is needed
+#   because, when a guard fails, the blackhole interp first rebuilds
+#   all the intermediate RPython frames; at that point it needs to
+#   call enter_code() on all intermediate RPython frames, so it does
+#   pattern matching to recognize frames and learn about unique_id.
+#
+# - The jitcode opcode 'rvmprof_code' doesn't produce any resop.  When
+#   meta-interpreting, it causes pyjitpl to call jit_enter_code(), and
+#   jit_leave_code().  There is logic to call jit_leave_code() even if
+#   we exit with an exception, even though there is no
+#   'catch_exception'.
+#
+# - When blackholing, the call to jit_enter_code() occurs imediately
+#   as described above.  For calling jit_leave_code(), we use the same
+#   logic, detecting when we need to call it even though there is no
+#   'catch_exception'.
+
+ at jit.oopspec("rvmprof.code(leaving, unique_id)")
+def jit_rvmprof_code(leaving, unique_id):
+    """Marker for the JIT.  Also called directly from the metainterp and
+    the blackhole interp."""
+    if not leaving:
+        enter_code(unique_id)    # ignore the return value
+    else:
+        s = vmprof_tl_stack.getraw()
+        assert s.c_value == unique_id
+        leave_code(s)
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,6 +1,6 @@
 import sys, os
 from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib import rposix
+from rpython.rlib import jit, rposix
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
@@ -162,19 +162,12 @@
     """
     if _hack_update_stack_untranslated:
         from rpython.rtyper.annlowlevel import llhelper
-        from rpython.rlib import jit
-        enter_code_untr = llhelper(lltype.Ptr(
+        enter_code = llhelper(lltype.Ptr(
             lltype.FuncType([lltype.Signed], cintf.PVMPROFSTACK)),
             cintf.enter_code)
-        leave_code_untr = llhelper(lltype.Ptr(
-            lltype.FuncType([cintf.PVMPROFSTACK, lltype.Signed], lltype.Void)),
+        leave_code = llhelper(lltype.Ptr(
+            lltype.FuncType([cintf.PVMPROFSTACK], lltype.Void)),
             cintf.leave_code)
-        @jit.oopspec("rvmprof.enter_code(unique_id)")
-        def enter_code(unique_id):
-            return enter_code_untr(unique_id)
-        @jit.oopspec("rvmprof.leave_code(s)")
-        def leave_code(s, unique_id):
-            leave_code_untr(s, unique_id)
     else:
         enter_code = cintf.enter_code
         leave_code = cintf.leave_code
@@ -185,13 +178,24 @@
         except cintf.VMProfPlatformUnsupported:
             return func
 
+        def decorated_jitted_function(unique_id, *args):
+            cintf.jit_rvmprof_code(0, unique_id)
+            res = func(*args)
+            cintf.jit_rvmprof_code(1, unique_id)   # no 'finally:', see cintf.py
+            return res
+        decorated_jitted_function._dont_inline_ = True
+
         def decorated_function(*args):
             unique_id = get_code_fn(*args)._vmprof_unique_id
-            x = enter_code(unique_id)
-            try:
-                return func(*args)
-            finally:
-                leave_code(x, unique_id)
+            if not jit.we_are_jitted():
+                x = enter_code(unique_id)
+                try:
+                    return func(*args)
+                finally:
+                    leave_code(x)
+            else:
+                return decorated_jitted_function(unique_id, *args)
+        decorated_function._always_inline_ = True
 
         decorated_function.__name__ = func.__name__ + '_rvmprof'
         return decorated_function

From pypy.commits at gmail.com  Sun Aug  7 04:44:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 01:44:03 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: in-progress
Message-ID: <57a6f4d3.11051c0a.c5268.d5f9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86053:2de09f51a345
Date: 2016-08-07 10:46 +0200
http://bitbucket.org/pypy/pypy/changeset/2de09f51a345/

Log:	in-progress

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -55,7 +55,6 @@
                 driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
                 if code.name == "main":
                     c = f(codes, codes[1], 1, c)
-                    driver.can_enter_jit(code=code, c=c, i=i, codes=codes, n=n)
                 else:
                     llfn()
                     c -= 1
@@ -74,6 +73,46 @@
         assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
 
 
+    def test_leaving_with_exception(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        class MyExc(Exception):
+            def __init__(self, c):
+                self.c = c
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
+            i = 0
+            while i < n:
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if code.name == "main":
+                    try:
+                        f(codes, codes[1], 1, c)
+                    except MyExc as e:
+                        c = e.c
+                else:
+                    llfn()
+                    c -= 1
+                i += 1
+            raise MyExc(c)
+
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            try:
+                f(codes, codes[0], n, 8)
+            except MyExc as e:
+                return e.c
+
+        null = lltype.nullptr(cintf.VMPROFSTACK)
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+
+
     def test_one(self):
 #        py.test.skip("needs thread-locals in the JIT, which is only available "
 #                     "after translation")
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -2082,10 +2082,30 @@
             raise NotImplementedError(oopspec_name)
 
     def _handle_rvmprof_call(self, op, oopspec_name, args):
-        if oopspec_name != 'rvmprof.code':
+        if oopspec_name != 'rvmprof.jitted':
             raise NotImplementedError(oopspec_name)
-        c_leaving, v_uniqueid = args
-        return SpaceOperation('rvmprof_code', [c_leaving, v_uniqueid], None)
+        c_entering = Constant(0, lltype.Signed)
+        c_leaving  = Constant(1, lltype.Signed)
+        v_uniqueid = args[0]
+        op1 = SpaceOperation('rvmprof_code', [c_entering, v_uniqueid], None)
+        op2 = SpaceOperation('rvmprof_code', [c_leaving, v_uniqueid], None)
+        #
+        # fish fish inside the oopspec's graph for the ll_func pointer
+        block = op.args[0].value._obj.graph.startblock
+        while True:
+            assert len(block.exits) == 1
+            nextblock = block.exits[0].target
+            if nextblock.operations == ():
+                break
+            block = nextblock
+        last_op = block.operations[-1]
+        assert last_op.opname == 'direct_call'
+        c_ll_func = last_op.args[0]
+        #
+        args = [c_ll_func] + op.args[2:]
+        ops = self.rewrite_op_direct_call(SpaceOperation('direct_call',
+                                                         args, op.result))
+        return [op1] + ops + [op2]
 
     def rewrite_op_ll_read_timestamp(self, op):
         op1 = self.prepare_builtin_call(op, "ll_read_timestamp", [])
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -1115,23 +1115,22 @@
         from rpython.rlib.rvmprof import cintf
         class MyFakeCallControl(FakeCallControl):
             def guess_call_kind(self, op):
-                if '_code' in repr(op):
+                if 'jitted' in repr(op):
                     return 'builtin'
                 return 'residual'
         class X:
             pass
-        def g():
+        def g(x, y):
             debug.debug_print("foo")
             return X()
-        g._dont_inline_ = True
-        def f(x):
-            cintf.jit_rvmprof_code(0, x)
-            res = g()
-            cintf.jit_rvmprof_code(1, x)
-            return res
-        self.encoding_test(f, [42], """
+        @jit.oopspec("rvmprof.jitted(unique_id)")
+        def decorated_jitted_function(unique_id, *args):
+            return g(*args)
+        def f(id, x, y):
+            return decorated_jitted_function(id, x, y)
+        self.encoding_test(f, [42, 56, 74], """
             rvmprof_code $0, %i0
-            residual_call_r_r $<* fn g>, R[], <Descr> -> %r0
+            residual_call_ir_r $<* fn g>, I[%i1, %i2], R[], <Descr> -> %r0
             -live-
             rvmprof_code $1, %i0
             ref_return %r0
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -7,6 +7,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib import rthread, jit
+from rpython.rlib.objectmodel import we_are_translated
 
 class VMProfPlatformUnsupported(Exception):
     pass
@@ -96,6 +97,8 @@
     return s
 
 def leave_code(s):
+    if not we_are_translated():
+        assert vmprof_tl_stack.getraw() == s
     vmprof_tl_stack.setraw(s.c_next)
     lltype.free(s, flavor='raw')
 
@@ -107,40 +110,35 @@
 #   (It uses kind == VMPROF_JITTED_TAG and the VMPROFSTACK is allocated
 #   in the C stack.)
 #
-# - The jitcode for decorated_jitted_function() in rvmprof.py, if
-#   we_are_jitted() calls the oopspec'ed function jit_rvmprof_code(),
-#   which turns into a simple jitcode opcode.  The jitcode has a
-#   simple structure:
+# - The jitcode for decorated_jitted_function() in rvmprof.py is
+#   special-cased by jtransform.py to produce this:
 #
 #        rvmprof_code(0, unique_id)
-#        res = inline_call FUNC
+#        res = inline_call FUNC         <- for func(*args)
 #        rvmprof_code(1, unique_id)
+#        return res
 #
-#   with no catch_exception logic for a "finally:" block.  Instead the
-#   blackhole interp looks for this simple pattern.  This is needed
-#   because, when a guard fails, the blackhole interp first rebuilds
-#   all the intermediate RPython frames; at that point it needs to
-#   call enter_code() on all intermediate RPython frames, so it does
-#   pattern matching to recognize frames and learn about unique_id.
+#   There is no 'catch_exception', but the second 'rvmprof_code' is
+#   meant to be executed even in case there was an exception.  This is
+#   done by a special case in pyjitpl.py and blackhole.py.  The point
+#   is that the above simple pattern can be detected by the blackhole
+#   interp, when it first rebuilds all the intermediate RPython
+#   frames; at that point it needs to call jit_enter_code() on all
+#   intermediate RPython frames, so it does pattern matching to
+#   recognize when it must call that and with which 'unique_id' value.
 #
 # - The jitcode opcode 'rvmprof_code' doesn't produce any resop.  When
-#   meta-interpreting, it causes pyjitpl to call jit_enter_code(), and
-#   jit_leave_code().  There is logic to call jit_leave_code() even if
-#   we exit with an exception, even though there is no
-#   'catch_exception'.
-#
-# - When blackholing, the call to jit_enter_code() occurs imediately
-#   as described above.  For calling jit_leave_code(), we use the same
-#   logic, detecting when we need to call it even though there is no
-#   'catch_exception'.
+#   meta-interpreting, it causes pyjitpl to call jit_enter_code() or
+#   jit_leave_code().  As mentioned above, there is logic to call
+#   jit_leave_code() even if we exit with an exception, even though
+#   there is no 'catch_exception'.  There is similar logic inside
+#   the blackhole interpreter.
 
- at jit.oopspec("rvmprof.code(leaving, unique_id)")
-def jit_rvmprof_code(leaving, unique_id):
-    """Marker for the JIT.  Also called directly from the metainterp and
-    the blackhole interp."""
-    if not leaving:
-        enter_code(unique_id)    # ignore the return value
-    else:
-        s = vmprof_tl_stack.getraw()
-        assert s.c_value == unique_id
-        leave_code(s)
+
+def jit_enter_code(unique_id):
+    enter_code(unique_id)    # ignore the return value
+
+def jit_leave_code(unique_id):
+    s = vmprof_tl_stack.getraw()
+    assert s.c_value == unique_id
+    leave_code(s)
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -178,12 +178,9 @@
         except cintf.VMProfPlatformUnsupported:
             return func
 
+        @jit.oopspec("rvmprof.jitted(unique_id)")
         def decorated_jitted_function(unique_id, *args):
-            cintf.jit_rvmprof_code(0, unique_id)
-            res = func(*args)
-            cintf.jit_rvmprof_code(1, unique_id)   # no 'finally:', see cintf.py
-            return res
-        decorated_jitted_function._dont_inline_ = True
+            return func(*args)
 
         def decorated_function(*args):
             unique_id = get_code_fn(*args)._vmprof_unique_id
@@ -195,7 +192,6 @@
                     leave_code(x)
             else:
                 return decorated_jitted_function(unique_id, *args)
-        decorated_function._always_inline_ = True
 
         decorated_function.__name__ = func.__name__ + '_rvmprof'
         return decorated_function

From pypy.commits at gmail.com  Sun Aug  7 04:51:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 01:51:21 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: Finish the pyjitpl part
Message-ID: <57a6f689.56421c0a.10d91.cee2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86054:e39f9543e5b0
Date: 2016-08-07 10:53 +0200
http://bitbucket.org/pypy/pypy/changeset/e39f9543e5b0/

Log:	Finish the pyjitpl part

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2080,9 +2080,14 @@
                     frame.pc = target
                     raise ChangeFrame
                 if opcode == self.staticdata.op_rvmprof_code:
-                    # do the 'leave_code' for rvmprof, but then continue
-                    # popping frames
-                    import pdb;pdb.set_trace()
+                    # do the 'jit_rvmprof_code(1)' for rvmprof, but then
+                    # continue popping frames.  Decode jit_rvmprof_code
+                    # manually here.
+                    from rpython.rlib.rvmprof import cintf
+                    arg1 = frame.registers_i[ord(code[position+1])].getint()
+                    arg2 = frame.registers_i[ord(code[position+2])].getint()
+                    assert arg1 == 1
+                    cintf.jit_rvmprof_code(arg1, arg2)
             self.popframe()
         try:
             self.compile_exit_frame_with_exception(self.last_exc_box)
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -123,22 +123,22 @@
 #   done by a special case in pyjitpl.py and blackhole.py.  The point
 #   is that the above simple pattern can be detected by the blackhole
 #   interp, when it first rebuilds all the intermediate RPython
-#   frames; at that point it needs to call jit_enter_code() on all
+#   frames; at that point it needs to call jit_rvmprof_code(0) on all
 #   intermediate RPython frames, so it does pattern matching to
 #   recognize when it must call that and with which 'unique_id' value.
 #
 # - The jitcode opcode 'rvmprof_code' doesn't produce any resop.  When
-#   meta-interpreting, it causes pyjitpl to call jit_enter_code() or
-#   jit_leave_code().  As mentioned above, there is logic to call
-#   jit_leave_code() even if we exit with an exception, even though
-#   there is no 'catch_exception'.  There is similar logic inside
-#   the blackhole interpreter.
+#   meta-interpreting, it causes pyjitpl to call jit_rvmprof_code().
+#   As mentioned above, there is logic to call jit_rvmprof_code(1)
+#   even if we exit with an exception, even though there is no
+#   'catch_exception'.  There is similar logic inside the blackhole
+#   interpreter.
 
 
-def jit_enter_code(unique_id):
-    enter_code(unique_id)    # ignore the return value
-
-def jit_leave_code(unique_id):
-    s = vmprof_tl_stack.getraw()
-    assert s.c_value == unique_id
-    leave_code(s)
+def jit_rvmprof_code(leaving, unique_id):
+    if leaving == 0:
+        enter_code(unique_id)    # ignore the return value
+    else:
+        s = vmprof_tl_stack.getraw()
+        assert s.c_value == unique_id
+        leave_code(s)

From pypy.commits at gmail.com  Sun Aug  7 05:09:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 02:09:36 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: Blackhole interp:
 in-progress
Message-ID: <57a6fad0.82ddc20a.5e0f1.1a8e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86055:3f7d7ab3e4ff
Date: 2016-08-07 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/3f7d7ab3e4ff/

Log:	Blackhole interp: in-progress

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -111,76 +111,3 @@
         cintf.vmprof_tl_stack.setraw(null)
         self.meta_interp(main, [30], inline=True)
         assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
-
-
-    def test_one(self):
-#        py.test.skip("needs thread-locals in the JIT, which is only available "
-#                     "after translation")
-        visited = []
-
-        def helper():
-            trace = []
-            stack = cintf.vmprof_tl_stack.getraw()
-            while stack:
-                trace.append((stack.c_kind, stack.c_value))
-                stack = stack.c_next
-            visited.append(trace)
-
-        llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
-
-        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
-
-        class CodeObj(object):
-            def __init__(self, name):
-                self.name = name
-
-        def get_code_fn(codes, code, arg, c):
-            return code
-
-        def get_name(code):
-            return "foo"
-
-        _get_vmprof().use_weaklist = False
-        register_code_object_class(CodeObj, get_name)
-
-        @vmprof_execute_code("main", get_code_fn,
-                             _hack_update_stack_untranslated=True)
-        def f(codes, code, n, c):
-            i = 0
-            while i < n:
-                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
-                if code.name == "main":
-                    c = f(codes, codes[1], 1, c)
-                    driver.can_enter_jit(code=code, c=c, i=i, codes=codes, n=n)
-                else:
-                    llfn()
-                    c -= 1
-                    if c < 0:
-                        llfn() # bridge
-                i += 1
-            return c
-
-        def main(n):
-            codes = [CodeObj("main"), CodeObj("not main")]
-            for code in codes:
-                register_code(code, get_name)
-            return f(codes, codes[0], n, 8)
-
-        class Hooks(jit.JitHookInterface):
-            def after_compile(self, debug_info):
-                self.raw_start = debug_info.asminfo.rawstart
-
-        hooks = Hooks()
-
-        null = lltype.nullptr(cintf.VMPROFSTACK)
-        cintf.vmprof_tl_stack.setraw(null)
-        self.meta_interp(main, [30], policy=JitPolicy(hooks), inline=True)
-        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
-        print visited
-        #v = set(visited)
-        #assert 0 in v
-        #v.remove(0)
-        #assert len(v) == 1
-        #assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
-        #assert cintf.vmprof_tl_stack.getraw() == null
-        # ^^^ make sure we didn't leave anything dangling
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -64,6 +64,7 @@
             assert self._insns[value] is None
             self._insns[value] = key
         self.op_catch_exception = insns.get('catch_exception/L', -1)
+        self.op_rvmprof_code = insns.get('rvmprof_code/ii', -1)
         #
         all_funcs = []
         for key in self._insns:
@@ -270,6 +271,7 @@
         self.dispatch_loop      = builder.dispatch_loop
         self.descrs             = builder.descrs
         self.op_catch_exception = builder.op_catch_exception
+        self.op_rvmprof_code    = builder.op_rvmprof_code
         self.count_interpreter  = count_interpreter
         #
         if we_are_translated():
@@ -376,6 +378,20 @@
         # no 'catch_exception' insn follows: just reraise
         reraise(e)
 
+    def handle_rvmprof_enter(self):
+        code = self.jitcode.code
+        position = self.position
+        opcode = ord(code[position])
+        if opcode == self.op_rvmprof_code:
+            arg1 = self.registers_i[ord(code[position + 1])]
+            arg2 = self.registers_i[ord(code[position + 2])]
+            if arg1 == 1:
+                # we are resuming at a position that will do a
+                # jit_rvmprof_code(1), when really executed.  That's a
+                # hint for the need for a jit_rvmprof_code(0).
+                from rpython.rlib.rvmprof import cintf
+                cintf.jit_rvmprof_code(0, arg2)
+
     def copy_constants(self, registers, constants):
         """Copy jitcode.constants[0] to registers[255],
                 jitcode.constants[1] to registers[254],
@@ -1503,7 +1519,8 @@
 
     @arguments("i", "i")
     def bhimpl_rvmprof_code(leaving, unique_id):
-        pass #import pdb;pdb.set_trace()
+        from rpython.rlib.rvmprof import cintf
+        cintf.jit_rvmprof_code(leaving, unique_id)
 
     # ----------
     # helpers to resume running in blackhole mode when a guard failed
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2084,8 +2084,8 @@
                     # continue popping frames.  Decode jit_rvmprof_code
                     # manually here.
                     from rpython.rlib.rvmprof import cintf
-                    arg1 = frame.registers_i[ord(code[position+1])].getint()
-                    arg2 = frame.registers_i[ord(code[position+2])].getint()
+                    arg1 = frame.registers_i[ord(code[position + 1])].getint()
+                    arg2 = frame.registers_i[ord(code[position + 2])].getint()
                     assert arg1 == 1
                     cintf.jit_rvmprof_code(arg1, arg2)
             self.popframe()
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1343,6 +1343,7 @@
         jitcode = jitcodes[jitcode_pos]
         curbh.setposition(jitcode, pc)
         resumereader.consume_one_section(curbh)
+        curbh.handle_rvmprof_enter()
     return curbh
 
 def force_from_resumedata(metainterp_sd, storage, deadframe, vinfo, ginfo):

From pypy.commits at gmail.com  Sun Aug  7 05:30:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 02:30:43 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: Trying but failing to
 write a test
Message-ID: <57a6ffc3.465d1c0a.1f06f.e16c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86056:cde94224c375
Date: 2016-08-07 11:32 +0200
http://bitbucket.org/pypy/pypy/changeset/cde94224c375/

Log:	Trying but failing to write a test

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -111,3 +111,44 @@
         cintf.vmprof_tl_stack.setraw(null)
         self.meta_interp(main, [30], inline=True)
         assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+
+
+    def test_leaving_with_exception_in_blackhole(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        class MyExc(Exception):
+            def __init__(self, c):
+                self.c = c
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
+            i = 0
+            while i < n:
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if code.name == "main":
+                    try:
+                        f(codes, codes[1], 1, c)
+                    except MyExc as e:
+                        c = e.c
+                else:
+                    llfn()
+                    c -= 1
+                i += 1
+            jit.promote(c + 5)     # failing guard
+            raise MyExc(c)
+
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            try:
+                f(codes, codes[0], n, 8)
+            except MyExc as e:
+                return e.c
+
+        null = lltype.nullptr(cintf.VMPROFSTACK)
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -375,6 +375,16 @@
                 target = ord(code[position+1]) | (ord(code[position+2])<<8)
                 self.position = target
                 return
+            if opcode == self.op_rvmprof_code:
+                import pdb;pdb.set_trace()
+                # do the 'jit_rvmprof_code(1)' for rvmprof, but then
+                # continue popping frames.  Decode jit_rvmprof_code
+                # manually here.
+                from rpython.rlib.rvmprof import cintf
+                arg1 = self.registers_i[ord(code[position + 1])]
+                arg2 = self.registers_i[ord(code[position + 2])]
+                assert arg1 == 1
+                cintf.jit_rvmprof_code(arg1, arg2)
         # no 'catch_exception' insn follows: just reraise
         reraise(e)
 

From pypy.commits at gmail.com  Sun Aug  7 05:54:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 02:54:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document how this pseudo-man-page is
 turned into the standard man-page
Message-ID: <57a7055f.0205c20a.69fbd.4598@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86057:d274b6cd9120
Date: 2016-08-07 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/d274b6cd9120/

Log:	Document how this pseudo-man-page is turned into the standard man-
	page format

diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 

From pypy.commits at gmail.com  Sun Aug  7 06:04:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 03:04:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: update
Message-ID: <57a707ab.c310c20a.42ec2.2bae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86058:dcc5844817d2
Date: 2016-08-07 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/dcc5844817d2/

Log:	update

diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.

From pypy.commits at gmail.com  Sun Aug  7 06:05:21 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 07 Aug 2016 03:05:21 -0700 (PDT)
Subject: [pypy-commit] pypy resource_warning: switch to pytest import
Message-ID: <57a707e1.c5aa1c0a.db8ad.ed30@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: resource_warning
Changeset: r86059:e03eb49532c1
Date: 2016-08-07 11:34 +0200
http://bitbucket.org/pypy/pypy/changeset/e03eb49532c1/

Log:	switch to pytest import

diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import py, os, errno
+import pytest, os, errno
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 
 def getfile(space):
@@ -26,7 +26,7 @@
 
     def setup_class(cls):
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+            str(pytest.ensuretemp("fileimpl").join("foo.txt")))
         cls.w_file = getfile(cls.space)
         cls.w_regex_search = cls.space.wrap(interp2app(regex_search))
 
@@ -388,7 +388,7 @@
         cls.old_read = os.read
 
         if cls.runappdirect:
-            py.test.skip("works with internals of _file impl on py.py")
+            pytest.skip("works with internals of _file impl on py.py")
         def read(fd, n=None):
             if fd != 424242:
                 return cls.old_read(fd, n)
@@ -427,9 +427,9 @@
 
     def setup_class(cls):
         if not cls.runappdirect:
-            py.test.skip("likely to deadlock when interpreted by py.py")
+            pytest.skip("likely to deadlock when interpreted by py.py")
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("concurrency.txt")))
+            str(pytest.ensuretemp("fileimpl").join("concurrency.txt")))
         cls.w_file = getfile(cls.space)
 
     def test_concurrent_writes(self):
@@ -540,7 +540,7 @@
 
     def setup_class(cls):
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+            str(pytest.ensuretemp("fileimpl").join("foo.txt")))
         cls.w_file = getfile(cls.space)
 
     def test___enter__(self):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,5 +1,5 @@
 import sys, os
-import py
+import pytest
 from pypy.tool.pytest.objspace import gettestobjspace
 from pypy.interpreter.gateway import interp2app
 from pypy.module._file.test.test_file import regex_search
@@ -14,8 +14,6 @@
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
     mod.path = udir.join('fd')
     mod.path.write('fo')
-    mod.raises = py.test.raises # make raises available from app-level tests
-    mod.skip = py.test.skip
 
 def test_gethostname():
     host = space.appexec([w_socket], "(_socket): return _socket.gethostname()")
@@ -43,7 +41,7 @@
     for host in ["localhost", "127.0.0.1", "::1"]:
         if host == "::1" and not ipv6:
             from pypy.interpreter.error import OperationError
-            with py.test.raises(OperationError):
+            with pytest.raises(OperationError):
                 space.appexec([w_socket, space.wrap(host)],
                               "(_socket, host): return _socket.gethostbyaddr(host)")
             continue
@@ -59,14 +57,14 @@
     assert space.unwrap(port) == 25
     # 1 arg version
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyname second argument is not optional before python 2.4")
+        pytest.skip("getservbyname second argument is not optional before python 2.4")
     port = space.appexec([w_socket, space.wrap(name)],
                         "(_socket, name): return _socket.getservbyname(name)")
     assert space.unwrap(port) == 25
 
 def test_getservbyport():
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyport does not exist before python 2.4")
+        pytest.skip("getservbyport does not exist before python 2.4")
     port = 25
     # 2 args version
     name = space.appexec([w_socket, space.wrap(port)],
@@ -99,7 +97,7 @@
 def test_fromfd():
     # XXX review
     if not hasattr(socket, 'fromfd'):
-        py.test.skip("No socket.fromfd on this platform")
+        pytest.skip("No socket.fromfd on this platform")
     orig_fd = path.open()
     fd = space.appexec([w_socket, space.wrap(orig_fd.fileno()),
             space.wrap(socket.AF_INET), space.wrap(socket.SOCK_STREAM),
@@ -159,7 +157,7 @@
 
 def test_pton_ntop_ipv4():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     tests = [
         ("123.45.67.89", "\x7b\x2d\x43\x59"),
         ("0.0.0.0", "\x00" * 4),
@@ -175,9 +173,9 @@
 
 def test_ntop_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -196,9 +194,9 @@
 
 def test_pton_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -217,7 +215,7 @@
         assert space.unwrap(w_packed) == packed
 
 def test_has_ipv6():
-    py.test.skip("has_ipv6 is always True on PyPy for now")
+    pytest.skip("has_ipv6 is always True on PyPy for now")
     res = space.appexec([w_socket], "(_socket): return _socket.has_ipv6")
     assert space.unwrap(res) == socket.has_ipv6
 
@@ -231,7 +229,7 @@
     w_l = space.appexec([w_socket, space.wrap(host), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, long(port))")
     assert space.unwrap(w_l) == info
-    py.test.skip("Unicode conversion is too slow")
+    pytest.skip("Unicode conversion is too slow")
     w_l = space.appexec([w_socket, space.wrap(unicode(host)), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, port)")
     assert space.unwrap(w_l) == info
@@ -252,7 +250,7 @@
 def test_addr_raw_packet():
     from pypy.module._socket.interp_socket import addr_as_object
     if not hasattr(rsocket._c, 'sockaddr_ll'):
-        py.test.skip("posix specific test")
+        pytest.skip("posix specific test")
     # HACK: To get the correct interface number of lo, which in most cases is 1,
     # but can be anything (i.e. 39), we need to call the libc function
     # if_nametoindex to get the correct index
@@ -690,11 +688,11 @@
 class AppTestNetlink:
     def setup_class(cls):
         if not hasattr(os, 'getpid'):
-            py.test.skip("AF_NETLINK needs os.getpid()")
+            pytest.skip("AF_NETLINK needs os.getpid()")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_NETLINK')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_NETLINK on this platform")
+            pytest.skip("no AF_NETLINK on this platform")
         cls.space = space
 
     def test_connect_to_kernel_netlink_routing_socket(self):
@@ -710,11 +708,11 @@
 class AppTestPacket:
     def setup_class(cls):
         if not hasattr(os, 'getuid') or os.getuid() != 0:
-            py.test.skip("AF_PACKET needs to be root for testing")
+            pytest.skip("AF_PACKET needs to be root for testing")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_PACKET')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_PACKET on this platform")
+            pytest.skip("no AF_PACKET on this platform")
         cls.space = space
 
     def test_convert_between_tuple_and_sockaddr_ll(self):

From pypy.commits at gmail.com  Sun Aug  7 06:05:23 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 07 Aug 2016 03:05:23 -0700 (PDT)
Subject: [pypy-commit] pypy resource_warning: skip resource warning tests
 under appdirect
Message-ID: <57a707e3.68adc20a.25dd3.37ac@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: resource_warning
Changeset: r86060:d74e414261fc
Date: 2016-08-07 11:35 +0200
http://bitbucket.org/pypy/pypy/changeset/d74e414261fc/

Log:	skip resource warning tests under appdirect

diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -218,6 +218,9 @@
         assert exc.value.filename == os.curdir
 
     def test_encoding_errors(self):
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            pytest.skip("pypy only test")
         import _file
 
         with self.file(self.temppath, "w") as f:
@@ -266,6 +269,7 @@
         if '__pypy__' in sys.builtin_module_names:
             assert repr(self.temppath) in g.getvalue()
 
+    @pytest.mark.skipif("config.option.runappdirect")
     def test_track_resources(self):
         import os, gc, sys, cStringIO
         if '__pypy__' not in sys.builtin_module_names:
@@ -308,6 +312,7 @@
         assert self.regex_search("WARNING: unclosed file: <open file .*>", msg)
         assert "Created at" not in msg
 
+    @pytest.mark.skipif("config.option.runappdirect")
     def test_track_resources_dont_crash(self):
         import os, gc, sys, cStringIO
         if '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -414,6 +414,7 @@
         if os.name != 'nt':
             raises(OSError, os.close, fileno)
 
+    @pytest.mark.skipif("config.option.runappdirect")
     def test_track_resources(self):
         import os, gc, sys, cStringIO
         import _socket

From pypy.commits at gmail.com  Sun Aug  7 06:05:27 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 07 Aug 2016 03:05:27 -0700 (PDT)
Subject: [pypy-commit] pypy resource_warning: add a paragraph about -X
 track-resources to cpython_differences. Also add the
Message-ID: <57a707e7.0dc11c0a.fa8c3.f01c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: resource_warning
Changeset: r86062:9a4bd1d4dfc7
Date: 2016-08-07 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/9a4bd1d4dfc7/

Log:	add a paragraph about -X track-resources to cpython_differences.
	Also add the option to the man-page

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -48,6 +48,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 

From pypy.commits at gmail.com  Sun Aug  7 06:05:25 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 07 Aug 2016 03:05:25 -0700 (PDT)
Subject: [pypy-commit] pypy resource_warning: add whatsnew entry
Message-ID: <57a707e5.c2f3c20a.a80d5.3c3e@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: resource_warning
Changeset: r86061:f12f10f3fd41
Date: 2016-08-07 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/f12f10f3fd41/

Log:	add whatsnew entry

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -119,3 +119,8 @@
 ``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
 importantly, CFFI calls that take directly a string as argument don't
 copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.

From pypy.commits at gmail.com  Sun Aug  7 06:24:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 03:24:11 -0700 (PDT)
Subject: [pypy-commit] cffi default: Eventually,
 replace this UserWarning with an error
Message-ID: <57a70c4b.4317c20a.5ff9b.42bf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2736:18cdf37d6b26
Date: 2016-08-07 12:25 +0200
http://bitbucket.org/cffi/cffi/changeset/18cdf37d6b26/

Log:	Eventually, replace this UserWarning with an error

diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")

From pypy.commits at gmail.com  Sun Aug  7 06:24:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 03:24:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/18cdf37d6b26
Message-ID: <57a70c73.0dc11c0a.fa8c3.f78e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86063:962a5a4ad08e
Date: 2016-08-07 12:26 +0200
http://bitbucket.org/pypy/pypy/changeset/962a5a4ad08e/

Log:	update to cffi/18cdf37d6b26

diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")

From pypy.commits at gmail.com  Sun Aug  7 08:35:55 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 07 Aug 2016 05:35:55 -0700 (PDT)
Subject: [pypy-commit] pypy resource_warning: close to-be-merged branch
Message-ID: <57a72b2b.a710c20a.3fa53.700a@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: resource_warning
Changeset: r86064:64bba3428b02
Date: 2016-08-07 14:33 +0200
http://bitbucket.org/pypy/pypy/changeset/64bba3428b02/

Log:	close to-be-merged branch


From pypy.commits at gmail.com  Sun Aug  7 08:35:57 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 07 Aug 2016 05:35:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge resource_warning:
Message-ID: <57a72b2d.915c1c0a.c121.2511@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86065:16e118636641
Date: 2016-08-07 14:35 +0200
http://bitbucket.org/pypy/pypy/changeset/16e118636641/

Log:	merge resource_warning:

	adds a new commandline option -X track-resources that will produce a
	ResourceWarning when the GC closes a file or socket. The traceback
	for the place where the file or socket was allocated is given as
	well

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -51,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -119,3 +119,8 @@
 ``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
 importantly, CFFI calls that take directly a string as argument don't
 copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -24,11 +24,15 @@
 -V     : print the Python version number and exit (also --version)
 -W arg : warning control; arg is action:message:category:module:lineno
          also PYTHONWARNINGS=arg
+-X arg : set implementation-specific option
 file   : program read from script file
 -      : program read from stdin (default; interactive mode if a tty)
 arg ...: arguments passed to program in sys.argv[1:]
+
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
+-X track-resources : track the creation of files and sockets and display
+                     a warning if they are not closed explicitly
 """
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
@@ -229,6 +233,14 @@
         import pypyjit
         pypyjit.set_param(jitparam)
 
+def set_runtime_options(options, Xparam, *args):
+    if Xparam == 'track-resources':
+        sys.pypy_set_track_resources(True)
+    else:
+        print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),)
+        print >> sys.stderr, '[options] can be: track-resources'
+        raise SystemExit
+
 class CommandLineError(Exception):
     pass
 
@@ -404,6 +416,7 @@
     '--info':    (print_info,      None),
     '--jit':     (set_jit_option,  Ellipsis),
     '-funroll-loops': (funroll_loops, None),
+    '-X':        (set_runtime_options, Ellipsis),
     '--':        (end_options,     None),
     }
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1764,6 +1764,40 @@
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
         """)
 
+    def resource_warning(self, w_msg, w_tb):
+        self.appexec([w_msg, w_tb],
+                     """(msg, tb):
+            import sys
+            print >> sys.stderr, msg
+            if tb:
+                print >> sys.stderr, "Created at (most recent call last):"
+                print >> sys.stderr, tb
+        """)
+
+    def format_traceback(self):
+        # we need to disable track_resources before calling the traceback
+        # module. Else, it tries to open more files to format the traceback,
+        # the file constructor will call space.format_traceback etc., in an
+        # inifite recursion
+        flag = self.sys.track_resources
+        self.sys.track_resources = False
+        try:
+            return self.appexec([],
+                         """():
+                import sys, traceback
+                # the "1" is because we don't want to show THIS code
+                # object in the traceback
+                try:
+                    f = sys._getframe(1)
+                except ValueError:
+                    # this happens if you call format_traceback at the very beginning
+                    # of startup, when there is no bottom code object
+                    return '<no stacktrace available>'
+                return "".join(traceback.format_stack(f))
+            """)
+        finally:
+            self.sys.track_resources = flag
+
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -220,6 +220,13 @@
         expected = {"no_user_site": True}
         self.check(['-c', 'pass'], {}, sys_argv=['-c'], run_command='pass', **expected)
 
+    def test_track_resources(self, monkeypatch):
+        myflag = [False]
+        def pypy_set_track_resources(flag):
+            myflag[0] = flag
+        monkeypatch.setattr(sys, 'pypy_set_track_resources', pypy_set_track_resources, raising=False)
+        self.check(['-X', 'track-resources'], {}, sys_argv=[''], run_stdin=True)
+        assert myflag[0] == True
 
 class TestInteraction:
     """
@@ -1074,4 +1081,3 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -427,3 +427,28 @@
         space.finish()
         # assert that we reach this point without getting interrupted
         # by the OperationError(NameError)
+
+    def test_format_traceback(self):
+        from pypy.tool.pytest.objspace import maketestobjspace
+        from pypy.interpreter.gateway import interp2app
+        #
+        def format_traceback(space):
+            return space.format_traceback()
+        #
+        space = maketestobjspace()
+        w_format_traceback = space.wrap(interp2app(format_traceback))
+        w_tb = space.appexec([w_format_traceback], """(format_traceback):
+            def foo():
+                return bar()
+            def bar():
+                return format_traceback()
+            return foo()
+        """)
+        tb = space.str_w(w_tb)
+        expected = '\n'.join([
+            '  File "?", line 6, in anonymous',  # this is the appexec code object
+            '  File "?", line 3, in foo',
+            '  File "?", line 5, in bar',
+            ''
+        ])
+        assert tb == expected
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -38,23 +38,33 @@
     errors   = None
     fd       = -1
     cffi_fileobj = None    # pypy/module/_cffi_backend
+    w_tb     = None  # String representation of the traceback at creation time
 
     newlines = 0     # Updated when the stream is closed
 
     def __init__(self, space):
         self.space = space
         self.register_finalizer(space)
+        if self.space.sys.track_resources:
+            self.w_tb = self.space.format_traceback()
 
     def _finalize_(self):
         # assume that the file and stream objects are only visible in the
         # thread that runs _finalize_, so no race condition should be
         # possible and no locking is done here.
-        if self.stream is not None:
-            try:
-                self.direct_close()
-            except StreamErrors as e:
-                operr = wrap_streamerror(self.space, e, self.w_name)
-                raise operr
+        if self.stream is None:
+            return
+        if self.space.sys.track_resources:
+            w_repr = self.space.repr(self)
+            str_repr = self.space.str_w(w_repr)
+            w_msg = self.space.wrap("WARNING: unclosed file: " + str_repr)
+            self.space.resource_warning(w_msg, self.w_tb)
+        #
+        try:
+            self.direct_close()
+        except StreamErrors as e:
+            operr = wrap_streamerror(self.space, e, self.w_name)
+            raise operr
 
     def fdopenstream(self, stream, fd, mode, w_name=None):
         self.fd = fd
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,6 @@
 from __future__ import with_statement
-import py, os, errno
+import pytest, os, errno
+from pypy.interpreter.gateway import interp2app, unwrap_spec
 
 def getfile(space):
     return space.appexec([], """():
@@ -10,13 +11,24 @@
             return file
     """)
 
+# the following function is used e.g. in test_resource_warning
+ at unwrap_spec(regex=str, s=str)
+def regex_search(space, regex, s):
+    import re
+    import textwrap
+    regex = textwrap.dedent(regex).strip()
+    m = re.search(regex, s)
+    m = bool(m)
+    return space.wrap(m)
+
 class AppTestFile(object):
     spaceconfig = dict(usemodules=("_file",))
 
     def setup_class(cls):
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+            str(pytest.ensuretemp("fileimpl").join("foo.txt")))
         cls.w_file = getfile(cls.space)
+        cls.w_regex_search = cls.space.wrap(interp2app(regex_search))
 
     def test_simple(self):
         f = self.file(self.temppath, "w")
@@ -206,6 +218,9 @@
         assert exc.value.filename == os.curdir
 
     def test_encoding_errors(self):
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            pytest.skip("pypy only test")
         import _file
 
         with self.file(self.temppath, "w") as f:
@@ -254,6 +269,71 @@
         if '__pypy__' in sys.builtin_module_names:
             assert repr(self.temppath) in g.getvalue()
 
+    @pytest.mark.skipif("config.option.runappdirect")
+    def test_track_resources(self):
+        import os, gc, sys, cStringIO
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("pypy specific test")
+        def fn(flag1, flag2, do_close=False):
+            sys.pypy_set_track_resources(flag1)
+            f = self.file(self.temppath, 'w')
+            sys.pypy_set_track_resources(flag2)
+            buf = cStringIO.StringIO()
+            preverr = sys.stderr
+            try:
+                sys.stderr = buf
+                if do_close:
+                    f.close()
+                del f
+                gc.collect() # force __del__ to be called
+            finally:
+                sys.stderr = preverr
+                sys.pypy_set_track_resources(False)
+            return buf.getvalue()
+
+        # check with track_resources disabled
+        assert fn(False, False) == ""
+        #
+        # check that we don't get the warning if we actually close the file
+        assert fn(False, False, do_close=True) == ""
+        #
+        # check with track_resources enabled
+        msg = fn(True, True)
+        assert self.regex_search(r"""
+        WARNING: unclosed file: <open file .*>
+        Created at \(most recent call last\):
+          File ".*", line .*, in test_track_resources
+          File ".*", line .*, in fn
+        """, msg)
+        #
+        # check with track_resources enabled in the destructor BUT with a
+        # file which was created when track_resources was disabled
+        msg = fn(False, True)
+        assert self.regex_search("WARNING: unclosed file: <open file .*>", msg)
+        assert "Created at" not in msg
+
+    @pytest.mark.skipif("config.option.runappdirect")
+    def test_track_resources_dont_crash(self):
+        import os, gc, sys, cStringIO
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("pypy specific test")
+        #
+        # try hard to create a code object whose co_filename points to an
+        # EXISTING file, so that traceback.py tries to open it when formatting
+        # the stacktrace
+        f = open(self.temppath, 'w')
+        f.close()
+        co = compile('open("%s")' % self.temppath, self.temppath, 'exec')
+        sys.pypy_set_track_resources(True)
+        try:
+            # this exec used to fail, because space.format_traceback tried to
+            # recurively open a file, causing an infinite recursion. For the
+            # purpose of this test, it is enough that it actually finishes
+            # without errors
+            exec co
+        finally:
+            sys.pypy_set_track_resources(False)
+
     def test_truncate(self):
         f = self.file(self.temppath, "w")
         f.write("foo")
@@ -313,7 +393,7 @@
         cls.old_read = os.read
 
         if cls.runappdirect:
-            py.test.skip("works with internals of _file impl on py.py")
+            pytest.skip("works with internals of _file impl on py.py")
         def read(fd, n=None):
             if fd != 424242:
                 return cls.old_read(fd, n)
@@ -352,9 +432,9 @@
 
     def setup_class(cls):
         if not cls.runappdirect:
-            py.test.skip("likely to deadlock when interpreted by py.py")
+            pytest.skip("likely to deadlock when interpreted by py.py")
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("concurrency.txt")))
+            str(pytest.ensuretemp("fileimpl").join("concurrency.txt")))
         cls.w_file = getfile(cls.space)
 
     def test_concurrent_writes(self):
@@ -465,7 +545,7 @@
 
     def setup_class(cls):
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+            str(pytest.ensuretemp("fileimpl").join("foo.txt")))
         cls.w_file = getfile(cls.space)
 
     def test___enter__(self):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -151,9 +151,23 @@
 
 
 class W_Socket(W_Root):
+    w_tb = None  # String representation of the traceback at creation time
+
     def __init__(self, space, sock):
+        self.space = space
         self.sock = sock
         register_socket(space, sock)
+        if self.space.sys.track_resources:
+            self.w_tb = self.space.format_traceback()
+            self.register_finalizer(space)
+
+    def _finalize_(self):
+        is_open = self.sock.fd >= 0
+        if is_open and self.space.sys.track_resources:
+            w_repr = self.space.repr(self)
+            str_repr = self.space.str_w(w_repr)
+            w_msg = self.space.wrap("WARNING: unclosed " + str_repr)
+            self.space.resource_warning(w_msg, self.w_tb)
 
     def get_type_w(self, space):
         return space.wrap(self.sock.type)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,6 +1,8 @@
 import sys, os
-import py
+import pytest
 from pypy.tool.pytest.objspace import gettestobjspace
+from pypy.interpreter.gateway import interp2app
+from pypy.module._file.test.test_file import regex_search
 from rpython.tool.udir import udir
 from rpython.rlib import rsocket
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -12,8 +14,6 @@
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
     mod.path = udir.join('fd')
     mod.path.write('fo')
-    mod.raises = py.test.raises # make raises available from app-level tests
-    mod.skip = py.test.skip
 
 def test_gethostname():
     host = space.appexec([w_socket], "(_socket): return _socket.gethostname()")
@@ -41,7 +41,7 @@
     for host in ["localhost", "127.0.0.1", "::1"]:
         if host == "::1" and not ipv6:
             from pypy.interpreter.error import OperationError
-            with py.test.raises(OperationError):
+            with pytest.raises(OperationError):
                 space.appexec([w_socket, space.wrap(host)],
                               "(_socket, host): return _socket.gethostbyaddr(host)")
             continue
@@ -57,14 +57,14 @@
     assert space.unwrap(port) == 25
     # 1 arg version
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyname second argument is not optional before python 2.4")
+        pytest.skip("getservbyname second argument is not optional before python 2.4")
     port = space.appexec([w_socket, space.wrap(name)],
                         "(_socket, name): return _socket.getservbyname(name)")
     assert space.unwrap(port) == 25
 
 def test_getservbyport():
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyport does not exist before python 2.4")
+        pytest.skip("getservbyport does not exist before python 2.4")
     port = 25
     # 2 args version
     name = space.appexec([w_socket, space.wrap(port)],
@@ -97,7 +97,7 @@
 def test_fromfd():
     # XXX review
     if not hasattr(socket, 'fromfd'):
-        py.test.skip("No socket.fromfd on this platform")
+        pytest.skip("No socket.fromfd on this platform")
     orig_fd = path.open()
     fd = space.appexec([w_socket, space.wrap(orig_fd.fileno()),
             space.wrap(socket.AF_INET), space.wrap(socket.SOCK_STREAM),
@@ -157,7 +157,7 @@
 
 def test_pton_ntop_ipv4():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     tests = [
         ("123.45.67.89", "\x7b\x2d\x43\x59"),
         ("0.0.0.0", "\x00" * 4),
@@ -173,9 +173,9 @@
 
 def test_ntop_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -194,9 +194,9 @@
 
 def test_pton_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -215,7 +215,7 @@
         assert space.unwrap(w_packed) == packed
 
 def test_has_ipv6():
-    py.test.skip("has_ipv6 is always True on PyPy for now")
+    pytest.skip("has_ipv6 is always True on PyPy for now")
     res = space.appexec([w_socket], "(_socket): return _socket.has_ipv6")
     assert space.unwrap(res) == socket.has_ipv6
 
@@ -229,7 +229,7 @@
     w_l = space.appexec([w_socket, space.wrap(host), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, long(port))")
     assert space.unwrap(w_l) == info
-    py.test.skip("Unicode conversion is too slow")
+    pytest.skip("Unicode conversion is too slow")
     w_l = space.appexec([w_socket, space.wrap(unicode(host)), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, port)")
     assert space.unwrap(w_l) == info
@@ -250,7 +250,7 @@
 def test_addr_raw_packet():
     from pypy.module._socket.interp_socket import addr_as_object
     if not hasattr(rsocket._c, 'sockaddr_ll'):
-        py.test.skip("posix specific test")
+        pytest.skip("posix specific test")
     # HACK: To get the correct interface number of lo, which in most cases is 1,
     # but can be anything (i.e. 39), we need to call the libc function
     # if_nametoindex to get the correct index
@@ -314,6 +314,7 @@
     def setup_class(cls):
         cls.space = space
         cls.w_udir = space.wrap(str(udir))
+        cls.w_regex_search = space.wrap(interp2app(regex_search))
 
     def teardown_class(cls):
         if not cls.runappdirect:
@@ -402,6 +403,64 @@
         if os.name != 'nt':
             raises(OSError, os.close, fileno)
 
+    def test_socket_track_resources(self):
+        import _socket, os, gc, sys, cStringIO
+        s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+        fileno = s.fileno()
+        assert s.fileno() >= 0
+        s.close()
+        assert s.fileno() < 0
+        s.close()
+        if os.name != 'nt':
+            raises(OSError, os.close, fileno)
+
+    @pytest.mark.skipif("config.option.runappdirect")
+    def test_track_resources(self):
+        import os, gc, sys, cStringIO
+        import _socket
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("pypy specific test")
+        #
+        def fn(flag1, flag2, do_close=False):
+            sys.pypy_set_track_resources(flag1)
+            mysock = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+            sys.pypy_set_track_resources(flag2)
+            buf = cStringIO.StringIO()
+            preverr = sys.stderr
+            try:
+                sys.stderr = buf
+                if do_close:
+                    mysock.close()
+                del mysock
+                gc.collect() # force __del__ to be called
+            finally:
+                sys.stderr = preverr
+                sys.pypy_set_track_resources(False)
+            return buf.getvalue()
+
+        # check with track_resources disabled
+        assert fn(False, False) == ""
+        #
+        # check that we don't get the warning if we actually closed the socket
+        msg = fn(True, True, do_close=True)
+        assert msg == ''
+        #
+        # check with track_resources enabled
+        msg = fn(True, True)
+        assert self.regex_search(r"""
+        WARNING: unclosed <socket object, .*>
+        Created at \(most recent call last\):
+          File ".*", line .*, in test_track_resources
+          File ".*", line .*, in fn
+        """, msg)
+        #
+        # track_resources is enabled after the construction of the socket. in
+        # this case, the socket is not registered for finalization at all, so
+        # we don't see a message
+        msg = fn(False, True)
+        assert msg == ''
+
+
     def test_socket_close_error(self):
         import _socket, os
         if os.name == 'nt':
@@ -630,11 +689,11 @@
 class AppTestNetlink:
     def setup_class(cls):
         if not hasattr(os, 'getpid'):
-            py.test.skip("AF_NETLINK needs os.getpid()")
+            pytest.skip("AF_NETLINK needs os.getpid()")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_NETLINK')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_NETLINK on this platform")
+            pytest.skip("no AF_NETLINK on this platform")
         cls.space = space
 
     def test_connect_to_kernel_netlink_routing_socket(self):
@@ -650,11 +709,11 @@
 class AppTestPacket:
     def setup_class(cls):
         if not hasattr(os, 'getuid') or os.getuid() != 0:
-            py.test.skip("AF_PACKET needs to be root for testing")
+            pytest.skip("AF_PACKET needs to be root for testing")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_PACKET')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_PACKET on this platform")
+            pytest.skip("no AF_PACKET on this platform")
         cls.space = space
 
     def test_convert_between_tuple_and_sockaddr_ll(self):
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -20,6 +20,7 @@
         self.defaultencoding = "ascii"
         self.filesystemencoding = None
         self.debug = True
+        self.track_resources = False
         self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
@@ -55,6 +56,8 @@
         '_current_frames'       : 'currentframes._current_frames',
         'setrecursionlimit'     : 'vm.setrecursionlimit',
         'getrecursionlimit'     : 'vm.getrecursionlimit',
+        'pypy_set_track_resources' : 'vm.set_track_resources',
+        'pypy_get_track_resources' : 'vm.get_track_resources',
         'setcheckinterval'      : 'vm.setcheckinterval',
         'getcheckinterval'      : 'vm.getcheckinterval',
         'exc_info'              : 'vm.exc_info',
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -61,6 +61,13 @@
     """
     return space.wrap(space.sys.recursionlimit)
 
+ at unwrap_spec(flag=bool)
+def set_track_resources(space, flag):
+    space.sys.track_resources = flag
+
+def get_track_resources(space):
+    return space.wrap(space.sys.track_resources)
+
 @unwrap_spec(interval=int)
 def setcheckinterval(space, interval):
     """Tell the Python interpreter to check for asynchronous events every
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -428,4 +428,5 @@
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
 FakeObjSpace.sys.dlopenflags = 123
+FakeObjSpace.sys.track_resources = False
 FakeObjSpace.builtin = FakeModule()

From pypy.commits at gmail.com  Sun Aug  7 13:46:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 10:46:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: typo
Message-ID: <57a773f4.88711c0a.509e1.8e14@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86066:20f28e81988a
Date: 2016-08-07 19:45 +0200
http://bitbucket.org/pypy/pypy/changeset/20f28e81988a/

Log:	typo

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -116,7 +116,7 @@
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would a too strong restriction to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).

From pypy.commits at gmail.com  Sun Aug  7 13:52:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 10:52:34 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: Tweak the test until it
 tests what it is supposed to.
Message-ID: <57a77562.09afc20a.998b.d209@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86067:39f3ff6ed5d7
Date: 2016-08-07 19:52 +0200
http://bitbucket.org/pypy/pypy/changeset/39f3ff6ed5d7/

Log:	Tweak the test until it tests what it is supposed to.

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -125,17 +125,20 @@
                              _hack_update_stack_untranslated=True)
         def f(codes, code, n, c):
             i = 0
-            while i < n:
+            while True:
                 driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if i >= n:
+                    break
+                i += 1
                 if code.name == "main":
                     try:
                         f(codes, codes[1], 1, c)
                     except MyExc as e:
                         c = e.c
+                    driver.can_enter_jit(code=code, c=c, i=i, codes=codes, n=n)
                 else:
                     llfn()
                     c -= 1
-                i += 1
             jit.promote(c + 5)     # failing guard
             raise MyExc(c)
 
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -376,7 +376,6 @@
                 self.position = target
                 return
             if opcode == self.op_rvmprof_code:
-                import pdb;pdb.set_trace()
                 # do the 'jit_rvmprof_code(1)' for rvmprof, but then
                 # continue popping frames.  Decode jit_rvmprof_code
                 # manually here.

From pypy.commits at gmail.com  Sun Aug  7 14:00:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 11:00:47 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: clean-ups
Message-ID: <57a7774f.81cb1c0a.a9579.e1bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86068:3b3cad7cd230
Date: 2016-08-07 20:00 +0200
http://bitbucket.org/pypy/pypy/changeset/3b3cad7cd230/

Log:	clean-ups

diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -376,8 +376,8 @@
                 self.position = target
                 return
             if opcode == self.op_rvmprof_code:
-                # do the 'jit_rvmprof_code(1)' for rvmprof, but then
-                # continue popping frames.  Decode jit_rvmprof_code
+                # call the 'jit_rvmprof_code(1)' for rvmprof, but then
+                # continue popping frames.  Decode the 'rvmprof_code' insn
                 # manually here.
                 from rpython.rlib.rvmprof import cintf
                 arg1 = self.registers_i[ord(code[position + 1])]
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -74,8 +74,6 @@
         self.parent_snapshot = None
         # counter for unrolling inlined loops
         self.unroll_iterations = 1
-        # rvmprof
-        self.rvmprof_unique_id = -1
 
     @specialize.arg(3)
     def copy_constants(self, registers, constants, ConstClass):
@@ -2080,8 +2078,8 @@
                     frame.pc = target
                     raise ChangeFrame
                 if opcode == self.staticdata.op_rvmprof_code:
-                    # do the 'jit_rvmprof_code(1)' for rvmprof, but then
-                    # continue popping frames.  Decode jit_rvmprof_code
+                    # call the 'jit_rvmprof_code(1)' for rvmprof, but then
+                    # continue popping frames.  Decode the 'rvmprof_code' insn
                     # manually here.
                     from rpython.rlib.rvmprof import cintf
                     arg1 = frame.registers_i[ord(code[position + 1])].getint()

From pypy.commits at gmail.com  Sun Aug  7 15:25:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 12:25:09 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: translation fix
Message-ID: <57a78b15.915c1c0a.c121.a7f1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86069:690fd43896cd
Date: 2016-08-07 20:33 +0100
http://bitbucket.org/pypy/pypy/changeset/690fd43896cd/

Log:	translation fix

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -184,6 +184,8 @@
 
         def decorated_function(*args):
             unique_id = get_code_fn(*args)._vmprof_unique_id
+            unique_id = rffi.cast(lltype.Signed, unique_id) 
+            # ^^^ removes the "known non-negative" hint for annotation
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)
                 try:

From pypy.commits at gmail.com  Sun Aug  7 16:24:39 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 07 Aug 2016 13:24:39 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-realloc: close branch to be merged
Message-ID: <57a79907.44ce1c0a.a84b7.bf47@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-realloc
Changeset: r86072:965b80b2b7c3
Date: 2016-08-07 23:21 +0300
http://bitbucket.org/pypy/pypy/changeset/965b80b2b7c3/

Log:	close branch to be merged


From pypy.commits at gmail.com  Sun Aug  7 16:24:41 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 07 Aug 2016 13:24:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge cpyext-realloc which implements
 PyObject_Realloc
Message-ID: <57a79909.81a2c20a.51926.589b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86073:4c630a5bbbdb
Date: 2016-08-07 23:22 +0300
http://bitbucket.org/pypy/pypy/changeset/4c630a5bbbdb/

Log:	merge cpyext-realloc which implements PyObject_Realloc

diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -21,6 +21,8 @@
                          flavor='raw',
                          add_memory_pressure=True)
 
+realloc = rffi.llexternal('realloc', [rffi.VOIDP, rffi.SIZE_T], rffi.VOIDP)
+
 @cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
@@ -28,7 +30,7 @@
                          flavor='raw',
                          add_memory_pressure=True)
     # XXX FIXME
-    return lltype.nullptr(rffi.VOIDP.TO)
+    return realloc(ptr, size)
 
 @cpython_api([rffi.VOIDP], lltype.Void)
 def PyObject_Free(space, ptr):
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -235,8 +235,9 @@
         assert type(x) is int
         assert x == -424344
 
-    @pytest.mark.skipif(True, reason='realloc not fully implemented')
     def test_object_realloc(self):
+        if not self.runappdirect:
+            skip('no untranslated support for realloc')
         module = self.import_extension('foo', [
             ("realloctest", "METH_NOARGS",
              """

From pypy.commits at gmail.com  Sun Aug  7 16:24:35 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 07 Aug 2016 13:24:35 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-realloc: merge default into branch
Message-ID: <57a79903.469d1c0a.9fd69.bbb4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-realloc
Changeset: r86070:a6addb94dc29
Date: 2016-08-07 23:15 +0300
http://bitbucket.org/pypy/pypy/changeset/a6addb94dc29/

Log:	merge default into branch

diff too long, truncating to 2000 out of 2979 lines

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -109,3 +109,18 @@
 .. branch: jitlog-exact-source-lines
 
 Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -24,11 +24,15 @@
 -V     : print the Python version number and exit (also --version)
 -W arg : warning control; arg is action:message:category:module:lineno
          also PYTHONWARNINGS=arg
+-X arg : set implementation-specific option
 file   : program read from script file
 -      : program read from stdin (default; interactive mode if a tty)
 arg ...: arguments passed to program in sys.argv[1:]
+
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
+-X track-resources : track the creation of files and sockets and display
+                     a warning if they are not closed explicitly
 """
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
@@ -229,6 +233,14 @@
         import pypyjit
         pypyjit.set_param(jitparam)
 
+def set_runtime_options(options, Xparam, *args):
+    if Xparam == 'track-resources':
+        sys.pypy_set_track_resources(True)
+    else:
+        print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),)
+        print >> sys.stderr, '[options] can be: track-resources'
+        raise SystemExit
+
 class CommandLineError(Exception):
     pass
 
@@ -404,6 +416,7 @@
     '--info':    (print_info,      None),
     '--jit':     (set_jit_option,  Ellipsis),
     '-funroll-loops': (funroll_loops, None),
+    '-X':        (set_runtime_options, Ellipsis),
     '--':        (end_options,     None),
     }
 
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -108,8 +108,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1156,3 +1156,22 @@
             counts = self.count_instructions(source)
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
+
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1764,6 +1764,40 @@
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
         """)
 
+    def resource_warning(self, w_msg, w_tb):
+        self.appexec([w_msg, w_tb],
+                     """(msg, tb):
+            import sys
+            print >> sys.stderr, msg
+            if tb:
+                print >> sys.stderr, "Created at (most recent call last):"
+                print >> sys.stderr, tb
+        """)
+
+    def format_traceback(self):
+        # we need to disable track_resources before calling the traceback
+        # module. Else, it tries to open more files to format the traceback,
+        # the file constructor will call space.format_traceback etc., in an
+        # inifite recursion
+        flag = self.sys.track_resources
+        self.sys.track_resources = False
+        try:
+            return self.appexec([],
+                         """():
+                import sys, traceback
+                # the "1" is because we don't want to show THIS code
+                # object in the traceback
+                try:
+                    f = sys._getframe(1)
+                except ValueError:
+                    # this happens if you call format_traceback at the very beginning
+                    # of startup, when there is no bottom code object
+                    return '<no stacktrace available>'
+                return "".join(traceback.format_stack(f))
+            """)
+        finally:
+            self.sys.track_resources = flag
+
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -220,6 +220,13 @@
         expected = {"no_user_site": True}
         self.check(['-c', 'pass'], {}, sys_argv=['-c'], run_command='pass', **expected)
 
+    def test_track_resources(self, monkeypatch):
+        myflag = [False]
+        def pypy_set_track_resources(flag):
+            myflag[0] = flag
+        monkeypatch.setattr(sys, 'pypy_set_track_resources', pypy_set_track_resources, raising=False)
+        self.check(['-X', 'track-resources'], {}, sys_argv=[''], run_stdin=True)
+        assert myflag[0] == True
 
 class TestInteraction:
     """
@@ -1074,4 +1081,3 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -427,3 +427,28 @@
         space.finish()
         # assert that we reach this point without getting interrupted
         # by the OperationError(NameError)
+
+    def test_format_traceback(self):
+        from pypy.tool.pytest.objspace import maketestobjspace
+        from pypy.interpreter.gateway import interp2app
+        #
+        def format_traceback(space):
+            return space.format_traceback()
+        #
+        space = maketestobjspace()
+        w_format_traceback = space.wrap(interp2app(format_traceback))
+        w_tb = space.appexec([w_format_traceback], """(format_traceback):
+            def foo():
+                return bar()
+            def bar():
+                return format_traceback()
+            return foo()
+        """)
+        tb = space.str_w(w_tb)
+        expected = '\n'.join([
+            '  File "?", line 6, in anonymous',  # this is the appexec code object
+            '  File "?", line 3, in foo',
+            '  File "?", line 5, in bar',
+            ''
+        ])
+        assert tb == expected
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.7.0"
+VERSION = "1.8.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -157,11 +157,13 @@
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
         try:
+            keepalives = [None] * len(args_w)    # None or strings
             for i in range(len(args_w)):
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                 w_obj = args_w[i]
                 argtype = self.fargs[i]
-                if argtype.convert_argument_from_object(data, w_obj):
+                if argtype.convert_argument_from_object(data, w_obj,
+                                                        keepalives, i):
                     # argtype is a pointer type, and w_obj a list/tuple/str
                     mustfree_max_plus_1 = i + 1
 
@@ -177,9 +179,13 @@
                 if isinstance(argtype, W_CTypePointer):
                     data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                     flag = get_mustfree_flag(data)
+                    raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                     if flag == 1:
-                        raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
+                    elif flag >= 4:
+                        value = keepalives[i]
+                        assert value is not None
+                        rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
             lltype.free(buffer, flavor='raw')
             keepalive_until_here(args_w)
         return w_res
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -83,7 +83,7 @@
         raise oefmt(space.w_TypeError, "cannot initialize cdata '%s'",
                     self.name)
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         self.convert_from_object(cdata, w_ob)
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,8 +14,8 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
+    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
     length = -1
 
     def __init__(self, space, size, extra, extra_position, ctitem,
@@ -28,6 +28,9 @@
         #  - for functions, it is the return type
         self.ctitem = ctitem
         self.can_cast_anything = could_cast_anything and ctitem.cast_anything
+        self.accept_str = (self.can_cast_anything or
+                            (ctitem.is_primitive_integer and
+                             ctitem.size == rffi.sizeof(lltype.Char)))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -70,9 +73,7 @@
                 pass
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
-        elif (self.can_cast_anything or
-              (self.ctitem.is_primitive_integer and
-               self.ctitem.size == rffi.sizeof(lltype.Char))):
+        elif self.accept_str:
             if not space.isinstance_w(w_ob, space.w_str):
                 raise self._convert_error("str or list or tuple", w_ob)
             s = space.str_w(w_ob)
@@ -260,8 +261,16 @@
         else:
             return lltype.nullptr(rffi.CCHARP.TO)
 
-    def _prepare_pointer_call_argument(self, w_init, cdata):
+    def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
         space = self.space
+        if self.accept_str and space.isinstance_w(w_init, space.w_str):
+            # special case to optimize strings passed to a "char *" argument
+            value = w_init.str_w(space)
+            keepalives[i] = value
+            buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
+            rffi.cast(rffi.CCHARPP, cdata)[0] = buf
+            return ord(buf_flag)    # 4, 5 or 6
+        #
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
@@ -297,10 +306,11 @@
         rffi.cast(rffi.CCHARPP, cdata)[0] = result
         return 1
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
         result = (not isinstance(w_ob, cdataobj.W_CData) and
-                  self._prepare_pointer_call_argument(w_ob, cdata))
+                  self._prepare_pointer_call_argument(w_ob, cdata,
+                                                      keepalives, i))
         if result == 0:
             self.convert_from_object(cdata, w_ob)
         set_mustfree_flag(cdata, result)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -353,7 +353,7 @@
 'array.array' or numpy arrays."""
         #
         w_ctchara = newtype._new_chara_type(self.space)
-        return func.from_buffer(self.space, w_ctchara, w_python_buffer)
+        return func._from_buffer(self.space, w_ctchara, w_python_buffer)
 
 
     @unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated
+from rpython.rlib import jit
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -132,17 +133,66 @@
         raise oefmt(space.w_TypeError,
                     "needs 'char[]', got '%s'", w_ctype.name)
     #
+    return _from_buffer(space, w_ctype, w_x)
+
+def _from_buffer(space, w_ctype, w_x):
     buf = _fetch_as_read_buffer(space, w_x)
-    try:
-        _cdata = buf.get_raw_address()
-    except ValueError:
-        raise oefmt(space.w_TypeError,
-                    "from_buffer() got a '%T' object, which supports the "
-                    "buffer interface but cannot be rendered as a plain "
-                    "raw address on PyPy", w_x)
+    if space.isinstance_w(w_x, space.w_str):
+        _cdata = get_raw_address_of_string(space, w_x)
+    else:
+        try:
+            _cdata = buf.get_raw_address()
+        except ValueError:
+            raise oefmt(space.w_TypeError,
+                        "from_buffer() got a '%T' object, which supports the "
+                        "buffer interface but cannot be rendered as a plain "
+                        "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
 
+# ____________________________________________________________
+
+class RawBytes(object):
+    def __init__(self, string):
+        self.ptr = rffi.str2charp(string, track_allocation=False)
+    def __del__(self):
+        rffi.free_charp(self.ptr, track_allocation=False)
+
+class RawBytesCache(object):
+    def __init__(self, space):
+        from pypy.interpreter.baseobjspace import W_Root
+        from rpython.rlib import rweakref
+        self.wdict = rweakref.RWeakKeyDictionary(W_Root, RawBytes)
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(space, w_x):
+    """Special case for ffi.from_buffer(string).  Returns a 'char *' that
+    is valid as long as the string object is alive.  Two calls to
+    ffi.from_buffer(same_string) are guaranteed to return the same pointer.
+    """
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rtyper.lltypesystem.rstr import STR
+    from rpython.rtyper.lltypesystem import llmemory
+    from rpython.rlib import rgc
+
+    cache = space.fromcache(RawBytesCache)
+    rawbytes = cache.wdict.get(w_x)
+    if rawbytes is None:
+        data = space.str_w(w_x)
+        if we_are_translated() and not rgc.can_move(data):
+            lldata = llstr(data)
+            data_start = (llmemory.cast_ptr_to_adr(lldata) +
+                          rffi.offsetof(STR, 'chars') +
+                          llmemory.itemoffsetof(STR.chars, 0))
+            data_start = rffi.cast(rffi.CCHARP, data_start)
+            data_start[len(data)] = '\x00'   # write the final extra null
+            return data_start
+        rawbytes = RawBytes(data)
+        cache.wdict.set(w_x, rawbytes)
+    return rawbytes.ptr
+
+# ____________________________________________________________
+
 
 def unsafe_escaping_ptr_for_ptr_or_array(w_cdata):
     if not w_cdata.ctype.is_nonfunc_pointer_or_array:
diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py
--- a/pypy/module/_cffi_backend/parse_c_type.py
+++ b/pypy/module/_cffi_backend/parse_c_type.py
@@ -97,11 +97,8 @@
                                   [rffi.INT], rffi.CCHARP)
 
 def parse_c_type(info, input):
-    p_input = rffi.str2charp(input)
-    try:
+    with rffi.scoped_view_charp(input) as p_input:
         res = ll_parse_c_type(info, p_input)
-    finally:
-        rffi.free_charp(p_input)
     return rffi.cast(lltype.Signed, res)
 
 NULL_CTX = lltype.nullptr(PCTX.TO)
@@ -130,15 +127,13 @@
     return rffi.getintfield(src_ctx, 'c_num_types')
 
 def search_in_globals(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_globals(ctx, c_name,
-                                  rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_globals(ctx, c_name,
+                                      rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
 
 def search_in_struct_unions(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_struct_unions(ctx, c_name,
-                                        rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_struct_unions(ctx, c_name,
+                                            rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3330,13 +3330,18 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharA = new_array_type(BCharP, None)
-    py.test.raises(TypeError, from_buffer, BCharA, b"foo")
+    p1 = from_buffer(BCharA, b"foo")
+    assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # from_buffer(buffer(b"foo")) does not work, because it's not
+        # implemented on pypy; only from_buffer(b"foo") works.
         py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
         py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
     try:
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -38,23 +38,33 @@
     errors   = None
     fd       = -1
     cffi_fileobj = None    # pypy/module/_cffi_backend
+    w_tb     = None  # String representation of the traceback at creation time
 
     newlines = 0     # Updated when the stream is closed
 
     def __init__(self, space):
         self.space = space
         self.register_finalizer(space)
+        if self.space.sys.track_resources:
+            self.w_tb = self.space.format_traceback()
 
     def _finalize_(self):
         # assume that the file and stream objects are only visible in the
         # thread that runs _finalize_, so no race condition should be
         # possible and no locking is done here.
-        if self.stream is not None:
-            try:
-                self.direct_close()
-            except StreamErrors as e:
-                operr = wrap_streamerror(self.space, e, self.w_name)
-                raise operr
+        if self.stream is None:
+            return
+        if self.space.sys.track_resources:
+            w_repr = self.space.repr(self)
+            str_repr = self.space.str_w(w_repr)
+            w_msg = self.space.wrap("WARNING: unclosed file: " + str_repr)
+            self.space.resource_warning(w_msg, self.w_tb)
+        #
+        try:
+            self.direct_close()
+        except StreamErrors as e:
+            operr = wrap_streamerror(self.space, e, self.w_name)
+            raise operr
 
     def fdopenstream(self, stream, fd, mode, w_name=None):
         self.fd = fd
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,6 @@
 from __future__ import with_statement
-import py, os, errno
+import pytest, os, errno
+from pypy.interpreter.gateway import interp2app, unwrap_spec
 
 def getfile(space):
     return space.appexec([], """():
@@ -10,13 +11,24 @@
             return file
     """)
 
+# the following function is used e.g. in test_resource_warning
+ at unwrap_spec(regex=str, s=str)
+def regex_search(space, regex, s):
+    import re
+    import textwrap
+    regex = textwrap.dedent(regex).strip()
+    m = re.search(regex, s)
+    m = bool(m)
+    return space.wrap(m)
+
 class AppTestFile(object):
     spaceconfig = dict(usemodules=("_file",))
 
     def setup_class(cls):
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+            str(pytest.ensuretemp("fileimpl").join("foo.txt")))
         cls.w_file = getfile(cls.space)
+        cls.w_regex_search = cls.space.wrap(interp2app(regex_search))
 
     def test_simple(self):
         f = self.file(self.temppath, "w")
@@ -206,6 +218,9 @@
         assert exc.value.filename == os.curdir
 
     def test_encoding_errors(self):
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            pytest.skip("pypy only test")
         import _file
 
         with self.file(self.temppath, "w") as f:
@@ -254,6 +269,71 @@
         if '__pypy__' in sys.builtin_module_names:
             assert repr(self.temppath) in g.getvalue()
 
+    @pytest.mark.skipif("config.option.runappdirect")
+    def test_track_resources(self):
+        import os, gc, sys, cStringIO
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("pypy specific test")
+        def fn(flag1, flag2, do_close=False):
+            sys.pypy_set_track_resources(flag1)
+            f = self.file(self.temppath, 'w')
+            sys.pypy_set_track_resources(flag2)
+            buf = cStringIO.StringIO()
+            preverr = sys.stderr
+            try:
+                sys.stderr = buf
+                if do_close:
+                    f.close()
+                del f
+                gc.collect() # force __del__ to be called
+            finally:
+                sys.stderr = preverr
+                sys.pypy_set_track_resources(False)
+            return buf.getvalue()
+
+        # check with track_resources disabled
+        assert fn(False, False) == ""
+        #
+        # check that we don't get the warning if we actually close the file
+        assert fn(False, False, do_close=True) == ""
+        #
+        # check with track_resources enabled
+        msg = fn(True, True)
+        assert self.regex_search(r"""
+        WARNING: unclosed file: <open file .*>
+        Created at \(most recent call last\):
+          File ".*", line .*, in test_track_resources
+          File ".*", line .*, in fn
+        """, msg)
+        #
+        # check with track_resources enabled in the destructor BUT with a
+        # file which was created when track_resources was disabled
+        msg = fn(False, True)
+        assert self.regex_search("WARNING: unclosed file: <open file .*>", msg)
+        assert "Created at" not in msg
+
+    @pytest.mark.skipif("config.option.runappdirect")
+    def test_track_resources_dont_crash(self):
+        import os, gc, sys, cStringIO
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("pypy specific test")
+        #
+        # try hard to create a code object whose co_filename points to an
+        # EXISTING file, so that traceback.py tries to open it when formatting
+        # the stacktrace
+        f = open(self.temppath, 'w')
+        f.close()
+        co = compile('open("%s")' % self.temppath, self.temppath, 'exec')
+        sys.pypy_set_track_resources(True)
+        try:
+            # this exec used to fail, because space.format_traceback tried to
+            # recurively open a file, causing an infinite recursion. For the
+            # purpose of this test, it is enough that it actually finishes
+            # without errors
+            exec co
+        finally:
+            sys.pypy_set_track_resources(False)
+
     def test_truncate(self):
         f = self.file(self.temppath, "w")
         f.write("foo")
@@ -313,7 +393,7 @@
         cls.old_read = os.read
 
         if cls.runappdirect:
-            py.test.skip("works with internals of _file impl on py.py")
+            pytest.skip("works with internals of _file impl on py.py")
         def read(fd, n=None):
             if fd != 424242:
                 return cls.old_read(fd, n)
@@ -352,9 +432,9 @@
 
     def setup_class(cls):
         if not cls.runappdirect:
-            py.test.skip("likely to deadlock when interpreted by py.py")
+            pytest.skip("likely to deadlock when interpreted by py.py")
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("concurrency.txt")))
+            str(pytest.ensuretemp("fileimpl").join("concurrency.txt")))
         cls.w_file = getfile(cls.space)
 
     def test_concurrent_writes(self):
@@ -465,7 +545,7 @@
 
     def setup_class(cls):
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+            str(pytest.ensuretemp("fileimpl").join("foo.txt")))
         cls.w_file = getfile(cls.space)
 
     def test___enter__(self):
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -401,21 +401,20 @@
             _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
         from rpython.rlib import rwin32
 
-        charp = rffi.str2charp(buf)
-        written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                    flavor='raw')
-        try:
-            result = _WriteFile(
-                self.handle, rffi.ptradd(charp, offset),
-                size, written_ptr, rffi.NULL)
+        with rffi.scoped_view_charp(buf) as charp:
+            written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
+                                        flavor='raw')
+            try:
+                result = _WriteFile(
+                    self.handle, rffi.ptradd(charp, offset),
+                    size, written_ptr, rffi.NULL)
 
-            if (result == 0 and
-                rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
-                raise oefmt(space.w_ValueError,
-                            "Cannot send %d bytes over connection", size)
-        finally:
-            rffi.free_charp(charp)
-            lltype.free(written_ptr, flavor='raw')
+                if (result == 0 and
+                    rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
+                    raise oefmt(space.w_ValueError,
+                                "Cannot send %d bytes over connection", size)
+            finally:
+                lltype.free(written_ptr, flavor='raw')
 
     def do_recv_string(self, space, buflength, maxlength):
         from pypy.module._multiprocessing.interp_win32 import (
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -20,7 +20,8 @@
     def _getfunc(space, CDLL, w_name, w_argtypes, w_restype):
         argtypes_w, argtypes, w_restype, restype = unpack_argtypes(
             space, w_argtypes, w_restype)
-        if space.isinstance_w(w_name, space.w_str):
+        if (space.isinstance_w(w_name, space.w_str) or
+                space.isinstance_w(w_name, space.w_unicode)):
             name = space.str_w(w_name)
             try:
                 func = CDLL.cdll.getpointer(name, argtypes, restype,
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -151,9 +151,23 @@
 
 
 class W_Socket(W_Root):
+    w_tb = None  # String representation of the traceback at creation time
+
     def __init__(self, space, sock):
+        self.space = space
         self.sock = sock
         register_socket(space, sock)
+        if self.space.sys.track_resources:
+            self.w_tb = self.space.format_traceback()
+            self.register_finalizer(space)
+
+    def _finalize_(self):
+        is_open = self.sock.fd >= 0
+        if is_open and self.space.sys.track_resources:
+            w_repr = self.space.repr(self)
+            str_repr = self.space.str_w(w_repr)
+            w_msg = self.space.wrap("WARNING: unclosed " + str_repr)
+            self.space.resource_warning(w_msg, self.w_tb)
 
     def get_type_w(self, space):
         return space.wrap(self.sock.type)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,6 +1,8 @@
 import sys, os
-import py
+import pytest
 from pypy.tool.pytest.objspace import gettestobjspace
+from pypy.interpreter.gateway import interp2app
+from pypy.module._file.test.test_file import regex_search
 from rpython.tool.udir import udir
 from rpython.rlib import rsocket
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -12,8 +14,6 @@
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
     mod.path = udir.join('fd')
     mod.path.write('fo')
-    mod.raises = py.test.raises # make raises available from app-level tests
-    mod.skip = py.test.skip
 
 def test_gethostname():
     host = space.appexec([w_socket], "(_socket): return _socket.gethostname()")
@@ -41,7 +41,7 @@
     for host in ["localhost", "127.0.0.1", "::1"]:
         if host == "::1" and not ipv6:
             from pypy.interpreter.error import OperationError
-            with py.test.raises(OperationError):
+            with pytest.raises(OperationError):
                 space.appexec([w_socket, space.wrap(host)],
                               "(_socket, host): return _socket.gethostbyaddr(host)")
             continue
@@ -57,14 +57,14 @@
     assert space.unwrap(port) == 25
     # 1 arg version
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyname second argument is not optional before python 2.4")
+        pytest.skip("getservbyname second argument is not optional before python 2.4")
     port = space.appexec([w_socket, space.wrap(name)],
                         "(_socket, name): return _socket.getservbyname(name)")
     assert space.unwrap(port) == 25
 
 def test_getservbyport():
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyport does not exist before python 2.4")
+        pytest.skip("getservbyport does not exist before python 2.4")
     port = 25
     # 2 args version
     name = space.appexec([w_socket, space.wrap(port)],
@@ -97,7 +97,7 @@
 def test_fromfd():
     # XXX review
     if not hasattr(socket, 'fromfd'):
-        py.test.skip("No socket.fromfd on this platform")
+        pytest.skip("No socket.fromfd on this platform")
     orig_fd = path.open()
     fd = space.appexec([w_socket, space.wrap(orig_fd.fileno()),
             space.wrap(socket.AF_INET), space.wrap(socket.SOCK_STREAM),
@@ -157,7 +157,7 @@
 
 def test_pton_ntop_ipv4():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     tests = [
         ("123.45.67.89", "\x7b\x2d\x43\x59"),
         ("0.0.0.0", "\x00" * 4),
@@ -173,9 +173,9 @@
 
 def test_ntop_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -194,9 +194,9 @@
 
 def test_pton_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -215,7 +215,7 @@
         assert space.unwrap(w_packed) == packed
 
 def test_has_ipv6():
-    py.test.skip("has_ipv6 is always True on PyPy for now")
+    pytest.skip("has_ipv6 is always True on PyPy for now")
     res = space.appexec([w_socket], "(_socket): return _socket.has_ipv6")
     assert space.unwrap(res) == socket.has_ipv6
 
@@ -229,7 +229,7 @@
     w_l = space.appexec([w_socket, space.wrap(host), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, long(port))")
     assert space.unwrap(w_l) == info
-    py.test.skip("Unicode conversion is too slow")
+    pytest.skip("Unicode conversion is too slow")
     w_l = space.appexec([w_socket, space.wrap(unicode(host)), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, port)")
     assert space.unwrap(w_l) == info
@@ -250,7 +250,7 @@
 def test_addr_raw_packet():
     from pypy.module._socket.interp_socket import addr_as_object
     if not hasattr(rsocket._c, 'sockaddr_ll'):
-        py.test.skip("posix specific test")
+        pytest.skip("posix specific test")
     # HACK: To get the correct interface number of lo, which in most cases is 1,
     # but can be anything (i.e. 39), we need to call the libc function
     # if_nametoindex to get the correct index
@@ -314,6 +314,7 @@
     def setup_class(cls):
         cls.space = space
         cls.w_udir = space.wrap(str(udir))
+        cls.w_regex_search = space.wrap(interp2app(regex_search))
 
     def teardown_class(cls):
         if not cls.runappdirect:
@@ -402,6 +403,64 @@
         if os.name != 'nt':
             raises(OSError, os.close, fileno)
 
+    def test_socket_track_resources(self):
+        import _socket, os, gc, sys, cStringIO
+        s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+        fileno = s.fileno()
+        assert s.fileno() >= 0
+        s.close()
+        assert s.fileno() < 0
+        s.close()
+        if os.name != 'nt':
+            raises(OSError, os.close, fileno)
+
+    @pytest.mark.skipif("config.option.runappdirect")
+    def test_track_resources(self):
+        import os, gc, sys, cStringIO
+        import _socket
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("pypy specific test")
+        #
+        def fn(flag1, flag2, do_close=False):
+            sys.pypy_set_track_resources(flag1)
+            mysock = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+            sys.pypy_set_track_resources(flag2)
+            buf = cStringIO.StringIO()
+            preverr = sys.stderr
+            try:
+                sys.stderr = buf
+                if do_close:
+                    mysock.close()
+                del mysock
+                gc.collect() # force __del__ to be called
+            finally:
+                sys.stderr = preverr
+                sys.pypy_set_track_resources(False)
+            return buf.getvalue()
+
+        # check with track_resources disabled
+        assert fn(False, False) == ""
+        #
+        # check that we don't get the warning if we actually closed the socket
+        msg = fn(True, True, do_close=True)
+        assert msg == ''
+        #
+        # check with track_resources enabled
+        msg = fn(True, True)
+        assert self.regex_search(r"""
+        WARNING: unclosed <socket object, .*>
+        Created at \(most recent call last\):
+          File ".*", line .*, in test_track_resources
+          File ".*", line .*, in fn
+        """, msg)
+        #
+        # track_resources is enabled after the construction of the socket. in
+        # this case, the socket is not registered for finalization at all, so
+        # we don't see a message
+        msg = fn(False, True)
+        assert msg == ''
+
+
     def test_socket_close_error(self):
         import _socket, os
         if os.name == 'nt':
@@ -630,11 +689,11 @@
 class AppTestNetlink:
     def setup_class(cls):
         if not hasattr(os, 'getpid'):
-            py.test.skip("AF_NETLINK needs os.getpid()")
+            pytest.skip("AF_NETLINK needs os.getpid()")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_NETLINK')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_NETLINK on this platform")
+            pytest.skip("no AF_NETLINK on this platform")
         cls.space = space
 
     def test_connect_to_kernel_netlink_routing_socket(self):
@@ -650,11 +709,11 @@
 class AppTestPacket:
     def setup_class(cls):
         if not hasattr(os, 'getuid') or os.getuid() != 0:
-            py.test.skip("AF_PACKET needs to be root for testing")
+            pytest.skip("AF_PACKET needs to be root for testing")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_PACKET')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_PACKET on this platform")
+            pytest.skip("no AF_PACKET on this platform")
         cls.space = space
 
     def test_convert_between_tuple_and_sockaddr_ll(self):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -135,7 +135,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
@@ -147,7 +147,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
@@ -181,7 +181,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
@@ -193,7 +193,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
@@ -228,7 +228,7 @@
 
         Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
         bound on the entropy contained in string."""
-        with rffi.scoped_str2charp(string) as buf:
+        with rffi.scoped_nonmovingbuffer(string) as buf:
             libssl_RAND_add(buf, len(string), entropy)
 
     def RAND_status(space):
diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -537,9 +537,8 @@
     releasegil=ts_helper,
     compilation_info=backend.eci)
 def c_charp2stdstring(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2stdstring(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2stdstring(charp)
     return result
 _c_stdstring2stdstring = rffi.llexternal(
     "cppyy_stdstring2stdstring",
diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -82,9 +82,8 @@
     releasegil=ts_helper,
     compilation_info=eci)
 def c_charp2TString(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2TString(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2TString(charp)
     return result
 _c_TString2TString = rffi.llexternal(
     "cppyy_TString2TString",
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -65,6 +65,7 @@
                 else:    # only other use is sring
                     n = len(obj._string)
                     assert raw_string == rffi.cast(rffi.CCHARP, 0)
+                    # XXX could use rffi.get_nonmovingbuffer_final_null()
                     raw_string = rffi.str2charp(obj._string)
                     data = rffi.cast(rffi.CCHARPP, data)
                     data[0] = raw_string
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,6 +11,9 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+        c_fdopen, c_fileno,
+        c_fopen)# for tests
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import NameManager
@@ -85,44 +88,32 @@
 assert CONST_WSTRING == rffi.CWCHARP
 
 # FILE* interface
-FILEP = rffi.COpaquePtr('FILE')
 
 if sys.platform == 'win32':
     dash = '_'
 else:
     dash = ''
-fileno = rffi.llexternal(dash + 'fileno', [FILEP], rffi.INT)
-fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
-fdopen = rffi.llexternal(dash + 'fdopen', [rffi.INT, CONST_STRING],
-                  FILEP, save_err=rffi.RFFI_SAVE_ERRNO)
 
-_fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
-    if not is_valid_fd(fileno(fp)):
+    if not is_valid_fd(c_fileno(fp)):
         return -1
-    return _fclose(fp)
+    return c_fclose(fp)
 
-_fwrite = rffi.llexternal('fwrite',
-                         [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                         rffi.SIZE_T)
 def fwrite(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fwrite(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fwrite(buf, sz, n, fp)
 
-_fread = rffi.llexternal('fread',
-                        [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                        rffi.SIZE_T)
 def fread(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fread(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fread(buf, sz, n, fp)
 
 _feof = rffi.llexternal('feof', [FILEP], rffi.INT)
 def feof(fp):
-    validate_fd(fileno(fp))
+    validate_fd(c_fileno(fp))
     return _feof(fp)
 
 def is_valid_fp(fp):
-    return is_valid_fd(fileno(fp))
+    return is_valid_fd(c_fileno(fp))
 
 pypy_decl = 'pypy_decl.h'
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -96,7 +96,8 @@
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
             py_str.c_ob_size, len(s))
-    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    with rffi.scoped_nonmovingbuffer(s) as s_ptr:
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
     py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -1,6 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rfile import c_setvbuf, _IONBF
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers, fdopen)
+    cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers, c_fdopen)
 from pypy.module.cpyext.pyobject import PyObject
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from pypy.interpreter.error import (OperationError, oefmt, 
@@ -64,11 +65,12 @@
     if (fd < 0 or not mode or mode[0] not in ['r', 'w', 'a', 'U'] or
         ('U' in mode and ('w' in mode or 'a' in mode))):
         raise oefmt(space.w_IOError, 'invalid fileno or mode') 
-    ret = fdopen(fd, mode)
+    ret = c_fdopen(fd, mode)
     if not ret:
         raise exception_from_saved_errno(space, space.w_IOError)
+    # XXX fix this once use-file-star-for-file lands
+    c_setvbuf(ret, lltype.nullptr(rffi.CCHARP.TO), _IONBF, 0)
     return ret
-        
 
 @cpython_api([FILEP, CONST_STRING, CONST_STRING, rffi.VOIDP], PyObject)
 def PyFile_FromFile(space, fp, name, mode, close):
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.eval import (
     Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP
+from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
@@ -130,19 +130,19 @@
     def test_run_file(self, space, api):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
-        fp = fopen(str(filepath), "rb")
+        fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_ZeroDivisionError
         api.PyErr_Clear()
 
         # try again, but with a closed file
-        fp = fopen(str(filepath), "rb")
-        os.close(fileno(fp))
+        fp = c_fopen(str(filepath), "rb")
+        os.close(c_fileno(fp))
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,4 @@
 from pypy.conftest import option
-from pypy.module.cpyext.api import fopen, fclose, fwrite
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
@@ -133,6 +132,15 @@
                     return PyLong_FromLong(0);
                 return PyLong_FromLong(ftell(fp));
              """),
+            ("read_10", "METH_O",
+             """
+                char s[10];
+                FILE * fp = PyFile_AsFile(args);
+                if (fp == NULL)
+                    return PyLong_FromLong(0);
+                fread(s, 1, 10, fp);
+                return PyLong_FromLong(ftell(fp));
+             """),
             ])
         filename = self.udir + "/_test_file"
         with open(filename, 'w') as fid:
@@ -142,5 +150,12 @@
             t_py = fid.tell()
             assert t_py == 80
             t_c = module.get_c_tell(fid)
-        assert t_c == t_py
+            assert t_c == t_py
+            print '-------- tell ',t_c
+            t_c = module.read_10(fid)
+            assert t_c == t_py + 10
+            print '-------- tell ',t_c
+            t_py = fid.tell()
+            assert t_c == t_py, 'after a fread, c level ftell(fp) %d but PyFile.tell() %d' % (t_c, t_py)
 
+
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -20,6 +20,7 @@
         self.defaultencoding = "ascii"
         self.filesystemencoding = None
         self.debug = True
+        self.track_resources = False
         self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
@@ -55,6 +56,8 @@
         '_current_frames'       : 'currentframes._current_frames',
         'setrecursionlimit'     : 'vm.setrecursionlimit',
         'getrecursionlimit'     : 'vm.getrecursionlimit',
+        'pypy_set_track_resources' : 'vm.set_track_resources',
+        'pypy_get_track_resources' : 'vm.get_track_resources',
         'setcheckinterval'      : 'vm.setcheckinterval',
         'getcheckinterval'      : 'vm.getcheckinterval',
         'exc_info'              : 'vm.exc_info',
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -61,6 +61,13 @@
     """
     return space.wrap(space.sys.recursionlimit)
 
+ at unwrap_spec(flag=bool)
+def set_track_resources(space, flag):
+    space.sys.track_resources = flag
+
+def get_track_resources(space):
+    return space.wrap(space.sys.track_resources)
+
 @unwrap_spec(interval=int)
 def setcheckinterval(space, interval):
     """Tell the Python interpreter to check for asynchronous events every
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -130,7 +130,7 @@
                 cls.module = str(udir.join('testownlib.dll'))
         else:
             subprocess.check_call(
-                'gcc testownlib.c -shared -fPIC -o testownlib.so',
+                'cc testownlib.c -shared -fPIC -o testownlib.so',
                 cwd=str(udir), shell=True)
             cls.module = str(udir.join('testownlib.so'))
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -852,9 +852,12 @@
     assert str(e2.value) == "foo0() takes no arguments (2 given)"
     assert str(e3.value) == "foo1() takes exactly one argument (0 given)"
     assert str(e4.value) == "foo1() takes exactly one argument (2 given)"
-    assert str(e5.value) == "foo2() takes exactly 2 arguments (0 given)"
-    assert str(e6.value) == "foo2() takes exactly 2 arguments (1 given)"
-    assert str(e7.value) == "foo2() takes exactly 2 arguments (3 given)"
+    assert str(e5.value) in ["foo2 expected 2 arguments, got 0",
+                             "foo2() takes exactly 2 arguments (0 given)"]
+    assert str(e6.value) in ["foo2 expected 2 arguments, got 1",
+                             "foo2() takes exactly 2 arguments (1 given)"]
+    assert str(e7.value) in ["foo2 expected 2 arguments, got 3",
+                             "foo2() takes exactly 2 arguments (3 given)"]
 
 def test_address_of_function():
     ffi = FFI()
@@ -1916,3 +1919,47 @@
     ffi.cdef("bool f(void);")
     lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
     assert lib.f() == 1
+
+def test_bool_in_cpp_2():
+    ffi = FFI()
+    ffi.cdef('int add(int a, int b);')
+    lib = verify(ffi, "test_bool_bug_cpp", '''
+        typedef bool _Bool;  /* there is a Windows header with this line */
+        int add(int a, int b)
+        {
+            return a + b;
+        }''', source_extension='.cpp')
+    c = lib.add(2, 3)
+    assert c == 5
+
+def test_struct_field_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[2]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+
+def test_function_arg_opaque():
+    py.test.skip("can currently declare a function with an opaque struct "
+                 "as argument, but AFAICT it's impossible to call it later")
+
+def test_function_returns_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a foo(int);")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_function_returns_opaque", "?")
+    assert str(e.value) == ("function foo: 'struct a' is used as result type,"
+                            " but is opaque")
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -133,6 +133,12 @@
         # You cannot assing character format codes as restype any longer
         raises(TypeError, setattr, f, "restype", "i")
 
+    def test_unicode_function_name(self):
+        f = dll[u'_testfunc_i_bhilfd']
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_int
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        assert result == 21
 
     def test_truncate_python_longs(self):
         f = dll._testfunc_i_bhilfd
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -428,4 +428,5 @@
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
 FakeObjSpace.sys.dlopenflags = 123
+FakeObjSpace.sys.track_resources = False
 FakeObjSpace.builtin = FakeModule()
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-# hypothesis is used for test generation on untranslated jit tests
+# hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -883,6 +883,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                               self.cpu.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -280,7 +280,7 @@
     concrete_type = '\x00'
 
     def __init__(self, basesize, itemsize, lendescr, flag, is_pure=False, concrete_type='\x00'):
-        self.basesize = basesize
+        self.basesize = basesize    # this includes +1 for STR
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
@@ -676,7 +676,7 @@
 
 def unpack_arraydescr(arraydescr):
     assert isinstance(arraydescr, ArrayDescr)
-    ofs = arraydescr.basesize
+    ofs = arraydescr.basesize    # this includes +1 for STR
     size = arraydescr.itemsize
     sign = arraydescr.is_item_signed()
     return size, ofs, sign
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -293,6 +293,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_load_or_indexed(op, op.getarg(0), op.getarg(1),
                                          itemsize, itemsize, basesize, NOT_SIGNED)
         elif opnum == rop.UNICODEGETITEM:
@@ -304,6 +305,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_store_or_indexed(op, op.getarg(0), op.getarg(1), op.getarg(2),
                                          itemsize, itemsize, basesize)
         elif opnum == rop.UNICODESETITEM:
diff --git a/rpython/jit/backend/llsupport/symbolic.py b/rpython/jit/backend/llsupport/symbolic.py
--- a/rpython/jit/backend/llsupport/symbolic.py
+++ b/rpython/jit/backend/llsupport/symbolic.py
@@ -29,7 +29,7 @@
 def get_array_token(T, translate_support_code):
     # T can be an array or a var-sized structure
     if translate_support_code:
-        basesize = llmemory.sizeof(T, 0)
+        basesize = llmemory.sizeof(T, 0)     # this includes +1 for STR
         if isinstance(T, lltype.Struct):
             SUBARRAY = getattr(T, T._arrayfld)
             itemsize = llmemory.sizeof(SUBARRAY.OF)
@@ -57,6 +57,7 @@
             assert carray.length.size == WORD
             ofs_length = before_array_part + carray.length.offset
         basesize = before_array_part + carray.items.offset
+        basesize += T._hints.get('extra_item_after_alloc', 0)  # +1 for STR
         carrayitem = ll2ctypes.get_ctypes_type(T.OF)
         itemsize = ctypes.sizeof(carrayitem)
     return basesize, itemsize, ofs_length
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -435,8 +435,10 @@
 def test_bytearray_descr():
     c0 = GcCache(False)
     descr = get_array_descr(c0, rstr.STR)   # for bytearray
+    # note that we get a basesize that has 1 extra byte for the final null char
+    # (only for STR)
     assert descr.flag == FLAG_UNSIGNED
-    assert descr.basesize == struct.calcsize("PP")         # hash, length
+    assert descr.basesize == struct.calcsize("PP") + 1     # hash, length, extra
     assert descr.lendescr.offset == struct.calcsize("P")   # hash
     assert not descr.is_array_of_pointers()
 
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -647,6 +647,9 @@
         """)
 
     def test_rewrite_assembler_newstr_newunicode(self):
+        # note: strdescr.basesize already contains the extra final character,
+        # so that's why newstr(14) is rounded up to 'basesize+15' and not
+        # 'basesize+16'.
         self.check_rewrite("""
             [i2]
             p0 = newstr(14)
@@ -657,12 +660,12 @@
         """, """
             [i2]
             p0 = call_malloc_nursery(                                \
-                      %(strdescr.basesize + 16 * strdescr.itemsize + \
+                      %(strdescr.basesize + 15 * strdescr.itemsize + \
                         unicodedescr.basesize + 10 * unicodedescr.itemsize)d)
             gc_store(p0, 0,  %(strdescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p0, %(strlendescr.offset)s, 14, %(strlendescr.field_size)s)
             gc_store(p0, 0,  0, %(strhashdescr.field_size)s)
-            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
+            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 15 * strdescr.itemsize)d)
             gc_store(p1, 0,  %(unicodedescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p1, %(unicodelendescr.offset)s, 10, %(unicodelendescr.field_size)s)
             gc_store(p1, 0,  0, %(unicodehashdescr.field_size)s)
@@ -1240,14 +1243,14 @@
         #               'i3 = gc_load_i(p0,i5,%(unicodedescr.itemsize)d)'],
         [True,  (4,),  'i3 = strgetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,1,'
-                       '%(strdescr.basesize)d,1)'],
+                       '%(strdescr.basesize-1)d,1)'],
         #[False, (4,),  'i3 = strgetitem(p0,i1)' '->'
-        #               'i5 = int_add(i1, %(strdescr.basesize)d);'
+        #               'i5 = int_add(i1, %(strdescr.basesize-1)d);'
         #               'i3 = gc_load_i(p0,i5,1)'],
         ## setitem str/unicode
         [True, (4,),  'i3 = strsetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,1,'
-                               '%(strdescr.basesize)d,1)'],
+                               '%(strdescr.basesize-1)d,1)'],
         [True, (2,4), 'i3 = unicodesetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,'
                                  '%(unicodedescr.itemsize)d,'
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -3,7 +3,7 @@
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
 from rpython.rlib.jit import promote, _get_virtualizable_token
-from rpython.rlib import jit_hooks, rposix
+from rpython.rlib import jit_hooks, rposix, rgc
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
 from rpython.jit.backend.detect_cpu import getcpuclass
@@ -11,7 +11,7 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 from rpython.config.config import ConfigError
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.rlib.rjitlog import rjitlog as jl
 
 
@@ -29,6 +29,7 @@
         # - floats neg and abs
         # - cast_int_to_float
         # - llexternal with macro=True
+        # - extra place for the zero after STR instances
 
         class BasicFrame(object):
             _virtualizable_ = ['i']
@@ -56,7 +57,7 @@
             return ("/home.py",0,0)
 
         jitdriver = JitDriver(greens = [],
-                              reds = ['total', 'frame', 'j'],
+                              reds = ['total', 'frame', 'prev_s', 'j'],
                               virtualizables = ['frame'],
                               get_location = get_location)
         def f(i, j):
@@ -68,9 +69,12 @@
             total = 0
             frame = Frame(i)
             j = float(j)
+            prev_s = rstr.mallocstr(16)
             while frame.i > 3:
-                jitdriver.can_enter_jit(frame=frame, total=total, j=j)
-                jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                jitdriver.can_enter_jit(frame=frame, total=total, j=j,
+                                        prev_s=prev_s)
+                jitdriver.jit_merge_point(frame=frame, total=total, j=j,
+                                          prev_s=prev_s)
                 _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
@@ -82,6 +86,11 @@
                 k = myabs1(myabs2(j))
                 if k - abs(j):  raise ValueError
                 if k - abs(-j): raise ValueError
+                s = rstr.mallocstr(16)
+                rgc.ll_write_final_null_char(s)
+                rgc.ll_write_final_null_char(prev_s)
+                if (frame.i & 3) == 0:
+                    prev_s = s
             return chr(total % 253)
         #
         class Virt2(object):
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -994,6 +994,7 @@
             basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                         self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             scale = 0
 
         self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale)
diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -2,7 +2,8 @@
 from rpython.rlib import jit
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rvmprof import cintf
+from rpython.rlib.rvmprof import cintf, vmprof_execute_code, register_code,\
+    register_code_object_class, _get_vmprof
 from rpython.jit.backend.x86.arch import WORD
 from rpython.jit.codewriter.policy import JitPolicy
 
@@ -14,6 +15,7 @@
 
         def helper():
             stack = cintf.vmprof_tl_stack.getraw()
+            print stack
             if stack:
                 # not during tracing
                 visited.append(stack.c_value)
@@ -22,15 +24,34 @@
 
         llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
 
-        driver = jit.JitDriver(greens=[], reds='auto')
+        driver = jit.JitDriver(greens=['code'], reds='auto')
 
-        def f(n):
+        class CodeObj(object):
+            pass
+
+        def get_code_fn(code, arg):
+            return code
+
+        def get_name(code):
+            return "foo"
+
+        register_code_object_class(CodeObj, get_name)
+
+        @vmprof_execute_code("main", get_code_fn)
+        def f(code, n):
             i = 0
             while i < n:
-                driver.jit_merge_point()
+                driver.jit_merge_point(code=code)
                 i += 1
                 llfn()
 
+        def main(n):
+            cintf.vmprof_tl_stack.setraw(null)   # make it empty
+            vmprof = _get_vmprof()
+            code = CodeObj()
+            register_code(code, get_name)
+            return f(code, n)
+
         class Hooks(jit.JitHookInterface):
             def after_compile(self, debug_info):
                 self.raw_start = debug_info.asminfo.rawstart
@@ -38,12 +59,12 @@
         hooks = Hooks()
 
         null = lltype.nullptr(cintf.VMPROFSTACK)
-        cintf.vmprof_tl_stack.setraw(null)   # make it empty
-        self.meta_interp(f, [10], policy=JitPolicy(hooks))
-        v = set(visited)
-        assert 0 in v
-        v.remove(0)
-        assert len(v) == 1
-        assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
-        assert cintf.vmprof_tl_stack.getraw() == null
+        self.meta_interp(main, [10], policy=JitPolicy(hooks))
+        print visited
+        #v = set(visited)
+        #assert 0 in v
+        #v.remove(0)
+        #assert len(v) == 1
+        #assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
+        #assert cintf.vmprof_tl_stack.getraw() == null
         # ^^^ make sure we didn't leave anything dangling
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1673,25 +1673,6 @@
         dest_addr = AddressLoc(base_loc, ofs_loc, scale, offset_loc.value)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
-    def genop_discard_strsetitem(self, op, arglocs):
-        base_loc, ofs_loc, val_loc = arglocs
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                              self.cpu.translate_support_code)
-        assert itemsize == 1
-        dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize)
-        self.mc.MOV8(dest_addr, val_loc.lowest8bits())
-
-    def genop_discard_unicodesetitem(self, op, arglocs):
-        base_loc, ofs_loc, val_loc = arglocs
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                              self.cpu.translate_support_code)
-        if itemsize == 4:
-            self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc)
-        elif itemsize == 2:
-            self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc)
-        else:
-            assert 0, itemsize
-
     # genop_discard_setfield_raw = genop_discard_setfield_gc
 
     def genop_math_read_timestamp(self, op, arglocs, resloc):
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1219,6 +1219,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                                   self.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self.assembler.load_effective_addr(ofsloc, ofs_items, scale,
                                            resloc, baseloc)
diff --git a/rpython/jit/backend/x86/test/test_rvmprof.py b/rpython/jit/backend/x86/test/test_rvmprof.py
--- a/rpython/jit/backend/x86/test/test_rvmprof.py
+++ b/rpython/jit/backend/x86/test/test_rvmprof.py
@@ -3,5 +3,5 @@
 from rpython.jit.backend.test.test_rvmprof import BaseRVMProfTest
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
 
-class TestFfiCall(Jit386Mixin, BaseRVMProfTest):
-    pass
\ No newline at end of file
+class TestRVMProfCall(Jit386Mixin, BaseRVMProfTest):
+    pass
diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py
--- a/rpython/jit/backend/zarch/opassembler.py
+++ b/rpython/jit/backend/zarch/opassembler.py
@@ -991,6 +991,7 @@
             basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                         self.cpu.translate_support_code)
             assert itemsize == 1

From pypy.commits at gmail.com  Sun Aug  7 16:24:37 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 07 Aug 2016 13:24:37 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-realloc: skip test if untranslated
Message-ID: <57a79905.c15e1c0a.917d6.b83a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-realloc
Changeset: r86071:6d1586833c45
Date: 2016-08-07 23:20 +0300
http://bitbucket.org/pypy/pypy/changeset/6d1586833c45/

Log:	skip test if untranslated

diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -236,6 +236,8 @@
         assert x == -424344
 
     def test_object_realloc(self):
+        if not self.runappdirect:
+            skip('no untranslated support for realloc')
         module = self.import_extension('foo', [
             ("realloctest", "METH_NOARGS",
              """

From pypy.commits at gmail.com  Sun Aug  7 16:24:43 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 07 Aug 2016 13:24:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <57a7990b.497bc20a.273c8.08ae@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86074:ebfe94bde5ab
Date: 2016-08-07 23:23 +0300
http://bitbucket.org/pypy/pypy/changeset/ebfe94bde5ab/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -124,3 +124,7 @@
 
 Add a new command line option -X track-resources which will produce
 ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc

From pypy.commits at gmail.com  Mon Aug  8 02:54:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 07 Aug 2016 23:54:41 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: Improve the test,
 fix by adding another missing case
Message-ID: <57a82cb1.031dc20a.af26a.a020@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86075:d58a138573e9
Date: 2016-08-08 08:54 +0200
http://bitbucket.org/pypy/pypy/changeset/d58a138573e9/

Log:	Improve the test, fix by adding another missing case

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -139,7 +139,8 @@
                 else:
                     llfn()
                     c -= 1
-            jit.promote(c + 5)     # failing guard
+            if c & 1:      # a failing guard
+                pass
             raise MyExc(c)
 
         def main(n):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1458,6 +1458,20 @@
         from rpython.rlib.rvmprof import cintf
         cintf.jit_rvmprof_code(leaving, box_unique_id.getint())
 
+    def handle_rvmprof_enter_on_resume(self):
+        code = self.bytecode
+        position = self.pc
+        opcode = ord(code[position])
+        if opcode == self.metainterp.staticdata.op_rvmprof_code:
+            arg1 = self.registers_i[ord(code[position + 1])].getint()
+            arg2 = self.registers_i[ord(code[position + 2])].getint()
+            if arg1 == 1:
+                # we are resuming at a position that will do a
+                # jit_rvmprof_code(1), when really executed.  That's a
+                # hint for the need for a jit_rvmprof_code(0).
+                from rpython.rlib.rvmprof import cintf
+                cintf.jit_rvmprof_code(0, arg2)
+
     # ------------------------------
 
     def setup_call(self, argboxes):
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1058,6 +1058,7 @@
         f.setup_resume_at_op(pc)
         resumereader.consume_boxes(f.get_current_position_info(),
                                    f.registers_i, f.registers_r, f.registers_f)
+        f.handle_rvmprof_enter_on_resume()
     return resumereader.liveboxes, virtualizable_boxes, virtualref_boxes
 
 

From pypy.commits at gmail.com  Mon Aug  8 07:35:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 08 Aug 2016 04:35:24 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: added unpack/pack test
 stressing the operation
Message-ID: <57a86e7c.c4ebc20a.86c3a.12c8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86076:b6f69665e955
Date: 2016-08-08 13:34 +0200
http://bitbucket.org/pypy/pypy/changeset/b6f69665e955/

Log:	added unpack/pack test stressing the operation

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -92,83 +92,35 @@
         self.VEC_DOUBLE_WORD_ONES = mem
 
     def emit_vec_load_f(self, op, arglocs, regalloc):
-        resloc, baseloc, indexloc, size_loc, ofs, integer_loc, aligned_loc = arglocs
+        resloc, baseloc, indexloc, size_loc, ofs, integer_loc = arglocs
         indexloc = self._apply_offset(indexloc, ofs)
         itemsize = size_loc.value
-        if itemsize == 4:
+        if integer_loc.value:
+            self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
+        elif itemsize == 4:
             self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
         elif itemsize == 8:
             self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
+        else:
+            not_implemented("vec_load_f itemsize %d" % itemsize)
 
-    def emit_vec_load_i(self, op, arglocs, regalloc):
-        resloc, baseloc, indexloc, size_loc, ofs, \
-            Vhiloc, Vloloc, Vploc, tloc = arglocs
-        indexloc = self._apply_offset(indexloc, ofs)
-        Vlo = Vloloc.value
-        Vhi = Vhiloc.value
-        self.mc.lvx(Vhi, indexloc.value, baseloc.value)
-        Vp = Vploc.value
-        t = tloc.value
-        if IS_BIG_ENDIAN:
-            self.mc.lvsl(Vp, indexloc.value, baseloc.value)
-        else:
-            self.mc.lvsr(Vp, indexloc.value, baseloc.value)
-        self.mc.addi(t, baseloc.value, 16)
-        self.mc.lvx(Vlo, indexloc.value, t)
-        if IS_BIG_ENDIAN:
-            self.mc.vperm(resloc.value, Vhi, Vlo, Vp)
-        else:
-            self.mc.vperm(resloc.value, Vlo, Vhi, Vp)
+    emit_vec_load_i = emit_vec_load_f
 
     def emit_vec_store(self, op, arglocs, regalloc):
         baseloc, indexloc, valueloc, sizeloc, baseofs, \
-            integer_loc, aligned_loc = arglocs
+            integer_loc = arglocs
         indexloc = self._apply_offset(indexloc, baseofs)
         assert baseofs.value == 0
         if integer_loc.value:
-            Vloloc = regalloc.vrm.get_scratch_reg(type=INT)
-            Vhiloc = regalloc.vrm.get_scratch_reg(type=INT)
-            Vploc = regalloc.vrm.get_scratch_reg(type=INT)
-            tloc = regalloc.rm.get_scratch_reg()
-            V1sloc = regalloc.vrm.get_scratch_reg(type=INT)
-            V1s = V1sloc.value
-            V0sloc = regalloc.vrm.get_scratch_reg(type=INT)
-            V0s = V0sloc.value
-            Vmaskloc = regalloc.vrm.get_scratch_reg(type=INT)
-            Vmask = Vmaskloc.value
-            Vlo = Vhiloc.value
-            Vhi = Vloloc.value
-            Vp = Vploc.value
-            t = tloc.value
-            Vs = valueloc.value
-            # UFF, that is a lot of code for storing unaligned!
-            # probably a lot of room for improvement (not locally,
-            # but in general for the algorithm)
-            self.mc.lvx(Vhi, indexloc.value, baseloc.value)
-            #self.mc.lvsr(Vp, indexloc.value, baseloc.value)
-            if IS_BIG_ENDIAN:
-                self.mc.lvsr(Vp, indexloc.value, baseloc.value)
-            else:
-                self.mc.lvsl(Vp, indexloc.value, baseloc.value)
-            self.mc.addi(t, baseloc.value, 16)
-            self.mc.lvx(Vlo, indexloc.value, t)
-            self.mc.vspltisb(V1s, -1)
-            self.mc.vspltisb(V0s, 0)
-            if IS_BIG_ENDIAN:
-                self.mc.vperm(Vmask, V0s, V1s, Vp)
-            else:
-                self.mc.vperm(Vmask, V1s, V0s, Vp)
-            self.mc.vperm(Vs, Vs, Vs, Vp)
-            self.mc.vsel(Vlo, Vs, Vlo, Vmask)
-            self.mc.vsel(Vhi, Vhi, Vs, Vmask)
-            self.mc.stvx(Vlo, indexloc.value, t)
-            self.mc.stvx(Vhi, indexloc.value, baseloc.value)
+            self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
         else:
             itemsize = sizeloc.value
             if itemsize == 4:
                 self.mc.stxvw4x(valueloc.value, indexloc.value, baseloc.value)
             elif itemsize == 8:
                 self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
+            else:
+                not_implemented("vec_store itemsize %d" % itemsize)
 
     def emit_vec_int_add(self, op, arglocs, regalloc):
         resloc, loc0, loc1, size_loc = arglocs
@@ -631,7 +583,6 @@
                not descr.is_array_of_structs()
         itemsize, ofs, _ = unpack_arraydescr(descr)
         integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
-        aligned = False
         args = op.getarglist()
         a0 = op.getarg(0)
         a1 = op.getarg(1)
@@ -639,28 +590,9 @@
         ofs_loc = self.ensure_reg(a1)
         result_loc = self.force_allocate_vector_reg(op)
         return [result_loc, base_loc, ofs_loc, imm(itemsize), imm(ofs),
-                imm(integer), imm(aligned)]
+                imm(integer)]
 
-    def _prepare_load_i(self, op):
-        descr = op.getdescr()
-        assert isinstance(descr, ArrayDescr)
-        assert not descr.is_array_of_pointers() and \
-               not descr.is_array_of_structs()
-        itemsize, ofs, _ = unpack_arraydescr(descr)
-        args = op.getarglist()
-        a0 = op.getarg(0)
-        a1 = op.getarg(1)
-        base_loc = self.ensure_reg(a0)
-        ofs_loc = self.ensure_reg(a1)
-        result_loc = self.force_allocate_vector_reg(op)
-        tloc = self.rm.get_scratch_reg()
-        Vhiloc = self.vrm.get_scratch_reg(type=INT)
-        Vloloc = self.vrm.get_scratch_reg(type=INT)
-        Vploc = self.vrm.get_scratch_reg(type=INT)
-        return [result_loc, base_loc, ofs_loc, imm(itemsize), imm(ofs),
-                Vhiloc, Vloloc, Vploc, tloc]
-
-    prepare_vec_load_i = _prepare_load_i
+    prepare_vec_load_i = _prepare_load
     prepare_vec_load_f = _prepare_load
 
     def prepare_vec_arith(self, op):
@@ -720,9 +652,8 @@
         valueloc = self.ensure_vector_reg(a2)
 
         integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
-        aligned = False
         return [baseloc, ofsloc, valueloc,
-                imm(itemsize), imm(ofs), imm(integer), imm(aligned)]
+                imm(itemsize), imm(ofs), imm(integer)]
 
     def prepare_vec_int_signext(self, op):
         assert isinstance(op, VectorOp)
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -531,6 +531,8 @@
                             self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
                             self.mc.UNPCKHPD(resloc, srcloc)
                         # if they are equal nothing is to be done
+        else:
+            not_implemented("pack/unpack for size %d", size)
 
     genop_vec_unpack_f = genop_vec_pack_f
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -18,6 +18,13 @@
 from rpython.rlib.objectmodel import (specialize, is_annotation_constant,
         always_inline)
 from rpython.jit.backend.detect_cpu import getcpuclass
+from rpython.jit.tool.oparser import parse
+from rpython.jit.metainterp.history import (AbstractFailDescr,
+                                            AbstractDescr,
+                                            BasicFailDescr, BasicFinalDescr,
+                                            JitCellToken, TargetToken,
+                                            ConstInt, ConstPtr,
+                                            Const, ConstFloat)
 
 CPU = getcpuclass()
 
@@ -78,7 +85,6 @@
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
     def setup_method(self, method):
-        import pdb; pdb.set_trace()
         if not self.supports_vector_ext():
             py.test.skip("this cpu %s has no implemented vector backend" % CPU)
 
@@ -718,5 +724,80 @@
         res = self.meta_interp(f, [22], vec_all=True, vec_guard_ratio=5)
         assert res == f(22)
 
+    def run_unpack(self, unpack, vector_type, assignments, float=True):
+        vars = {'v':0,'f':0,'i':0}
+        def newvar(type):
+            c = vars[type]
+            vars[type] = c + 1
+            if type == 'v':
+                return type + str(c) + vector_type
+            return type + str(c)
+        targettoken = TargetToken()
+        finaldescr = BasicFinalDescr(1)
+        args = []
+        args_values = []
+        pack = []
+        suffix = 'f' if float else 'i'
+        for var, vals in assignments.items():
+            v = newvar('v')
+            pack.append('%s = vec_%s()' % (v, suffix))
+            for i,val in enumerate(vals):
+                args_values.append(val)
+                f = newvar('f')
+                args.append(f)
+                count = 1
+                # create a new variable
+                vo = v
+                v = newvar('v')
+                pack.append('%s = vec_pack_%s(%s, %s, %d, %d)' % \
+                            (v, suffix, vo, f, i, count))
+            vars['x'] = v
+        packs = '\n        '.join(pack)
+        resvar = suffix + '{'+suffix+'}'
+        source = '''
+        [{args}]
+        label({args}, descr=targettoken)
+        {packs}
+        {unpack}
+        finish({resvar}, descr=finaldescr)
+        '''.format(args=','.join(args),packs=packs, unpack=unpack.format(**vars),
+                   resvar=resvar.format(**vars))
+        loop = parse(source, namespace={'targettoken': targettoken,
+                                        'finaldescr': finaldescr})
+
+        cpu = self.CPUClass(rtyper=None, stats=None)
+        cpu.setup_once()
+        #
+        looptoken = JitCellToken()
+        cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+        deadframe = cpu.execute_token(looptoken, *args_values)
+        print(source)
+        if float:
+            return cpu.get_float_value(deadframe, 0)
+        else:
+            return cpu.get_int_value(deadframe, 0)
+
+    def test_unpack(self):
+        # double unpack
+        assert self.run_unpack("f{f} = vec_unpack_f({x}, 0, 1)",
+                               "[2xf64]", {'x': (1.2,-1)}) == 1.2
+        assert self.run_unpack("f{f} = vec_unpack_f({x}, 1, 1)",
+                               "[2xf64]", {'x': (50.33,4321.0)}) == 4321.0
+        # int64
+        assert self.run_unpack("i{i} = vec_unpack_i({x}, 0, 1)",
+                               "[2xi64]", {'x': (11,12)}, float=False) == 11
+        assert self.run_unpack("i{i} = vec_unpack_i({x}, 1, 1)",
+                               "[2xi64]", {'x': (14,15)}, float=False) == 15
+
+        ## integer unpack (byte)
+        for i in range(16):
+            op = "i{i} = vec_unpack_i({x}, %d, 1)" % i
+            assert self.run_unpack(op, "[16xi8]", {'x': [127,1]*8}, float=False) == (127 if i%2==0 else 1)
+            if i < 8:
+                assert self.run_unpack(op, "[2xi16]", {'x': [2**15-1,0]*4}, float=False) == (2**15-1 if i%2==0 else 0)
+            if i < 4:
+                assert self.run_unpack(op, "[2xi32]", {'x': [2**31-1,0]*4}, float=False) == (2**31-1 if i%2==0 else 0)
+
+
 class TestLLtype(LLJitMixin, VectorizeTests):
     pass
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -299,6 +299,7 @@
             vecinfo.datatype = match.group(3)
             vecinfo.bytesize = int(match.group(4)) // 8
             resop._vec_debug_info = vecinfo
+            resop.bytesize = vecinfo.bytesize
             return var[:var.find('[')]
 
         vecinfo = VectorizationInfo(resop)

From pypy.commits at gmail.com  Mon Aug  8 12:01:34 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 08 Aug 2016 09:01:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Add test for asyncio checking a GIL
 initialization error on running "await asyncio.open_connection"
Message-ID: <57a8acde.c41f1c0a.c04bb.4532@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86077:3e98b6f5791a
Date: 2016-08-08 18:00 +0200
http://bitbucket.org/pypy/pypy/changeset/3e98b6f5791a/

Log:	Add test for asyncio checking a GIL initialization error on running
	"await asyncio.open_connection"

diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -0,0 +1,17 @@
+class AppTestAsyncIO(object):
+    
+    spaceconfig = dict(usemodules=["select","_socket","thread","signal","struct","_multiprocessing","array","_posixsubprocess","fcntl","unicodedata"])
+    
+    def setup_class(cls):
+        cls.space.appexec([], """():
+        import encodings.idna
+        import asyncio
+        async def f():
+            reader, writer = await asyncio.open_connection('example.com', 80)
+        
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(f())""")
+    
+    def test_gil_issue(self):
+        #needed to execute setup_call in the first place
+        assert 1==1
\ No newline at end of file

From pypy.commits at gmail.com  Mon Aug  8 12:48:02 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 08 Aug 2016 09:48:02 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix translation
Message-ID: <57a8b7c2.c3f0c20a.4a412.a6db@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86078:f12312dfba23
Date: 2016-06-30 19:35 +0200
http://bitbucket.org/pypy/pypy/changeset/f12312dfba23/

Log:	fix translation

diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -81,6 +81,7 @@
             debug_stop("jit-log-compiling-bridge")
         else:
             debug_start("jit-log-opt-bridge")
+            logops = None
             if have_debug_prints():
                 print_after_inputargs = ''
                 debug_print("# bridge out of Guard",

From pypy.commits at gmail.com  Mon Aug  8 12:48:05 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 08 Aug 2016 09:48:05 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix a bug around logging ops
Message-ID: <57a8b7c5.68adc20a.25dd3.a24e@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86079:464652309dcc
Date: 2016-06-30 19:35 +0200
http://bitbucket.org/pypy/pypy/changeset/464652309dcc/

Log:	fix a bug around logging ops

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1136,6 +1136,9 @@
                 assert isinstance(newdescr, GuardCompatibleDescr)
                 compat_cond = newdescr._compatibility_conditions
                 self.other_compat_conditions.append(compat_cond)
+        if not compat_cond:
+            assert self.fallback_jump_target == 0 # this can never happen twice
+            self.fallback_jump_target = -1
         asminfo = ResumeGuardDescr.compile_and_attach(
             self, metainterp, new_loop, orig_inputargs)
         # note that the backend will not patch the switch at all, so it is
@@ -1143,7 +1146,7 @@
         if compat_cond:
             compat_cond.jump_target = asminfo.asmaddr
         else:
-            assert self.fallback_jump_target == 0 # this can never happen twice
+            assert self.fallback_jump_target == -1 # this can never happen twice
             self.fallback_jump_target = asminfo.asmaddr
         return asminfo
 
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -89,7 +89,7 @@
                             "with", len(operations), "ops")
                 logops = self._make_log_operations(memo)
                 if isinstance(descr, GuardCompatibleDescr):
-                    if descr.fallback_jump_target == 0:
+                    if descr.fallback_jump_target != -1:
                         # this means it's the last attached guard
                         ccond = descr.other_compat_conditions[-1]
                         argrepr = logops.repr_of_arg(

From pypy.commits at gmail.com  Mon Aug  8 12:48:06 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 08 Aug 2016 09:48:06 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: since b116f09c4e9d we can no
 longer clear the last_quasi_immut_field_op in the GuardCompatibleDescr
Message-ID: <57a8b7c6.12331c0a.bb1d6.5509@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86080:9e5474cdc047
Date: 2016-06-30 19:48 +0200
http://bitbucket.org/pypy/pypy/changeset/9e5474cdc047/

Log:	since b116f09c4e9d we can no longer clear the
	last_quasi_immut_field_op in the GuardCompatibleDescr

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -138,7 +138,6 @@
         if not qmutdescr.is_still_valid_for(self.known_valid):
             return None, None
         copied_op.setarg(2, qmutdescr.constantfieldbox)
-        self.last_quasi_immut_field_op = None
         return copied_op, QuasiimmutGetfieldAndPureCallCondition(
                 op, qmutdescr, optimizer)
 
diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py
--- a/rpython/jit/metainterp/test/test_compatible.py
+++ b/rpython/jit/metainterp/test/test_compatible.py
@@ -538,7 +538,6 @@
         assert x < 30
         self.check_trace_count(7)
 
-
     def test_quasi_immutable_merge_short_preamble(self):
         from rpython.rlib.objectmodel import we_are_translated
         class C(object):
@@ -614,3 +613,64 @@
         self.check_resops(call_i=0)
 
 
+    def test_like_objects(self):
+        from rpython.rlib.objectmodel import we_are_translated
+        class Map(object):
+            _immutable_fields_ = ['version?']
+
+            def __init__(self):
+                self.version = Version()
+                self.dct = {}
+
+            def instantiate(self):
+                return Obj(self)
+
+            @jit.elidable_compatible(quasi_immut_field_name_for_second_arg='version')
+            def lookup_version(self, version, name):
+                return self.dct.get(name, -1)
+
+        class Version(object):
+            pass
+
+        class Obj(object):
+            def __init__(self, map):
+                self.map = map
+
+            def lookup(self, name):
+                map = self.map
+                assert isinstance(map, Map)
+                map = jit.hint(map, promote_compatible=True)
+                return map.lookup_version(name)
+
+        m1 = Map()
+        m1.dct['a'] = 1
+        m1.dct['b'] = 2
+        m2 = Map()
+        m2.dct['a'] = 1
+        m2.dct['b'] = 2
+        m2.dct['c'] = 5
+
+        p1 = m1.instantiate()
+        p2 = m2.instantiate()
+
+        driver = jit.JitDriver(greens = [], reds = ['n', 'res', 'p'])
+
+        def f(n, p):
+            res = 0
+            while n > 0:
+                driver.jit_merge_point(n=n, p=p, res=res)
+                res += p.lookup('a')
+                res += p.lookup('c')
+                res += p.lookup('b')
+                n -= 1
+            return res
+
+        def main(x):
+            res = f(100, p1)
+            res = f(100, p2)
+        main(True)
+        main(False)
+
+        self.meta_interp(main, [True], backendopt=True)
+        self.check_trace_count(2)
+        self.check_resops(call_i=0)

From pypy.commits at gmail.com  Mon Aug  8 12:48:08 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 08 Aug 2016 09:48:08 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix fake jit_debug ops
Message-ID: <57a8b7c8.53b81c0a.e11d7.57e9@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86081:12edeb760312
Date: 2016-07-06 17:49 +0200
http://bitbucket.org/pypy/pypy/changeset/12edeb760312/

Log:	fix fake jit_debug ops

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -183,6 +183,8 @@
 
     def repr_of_conditions_as_jit_debug(self, argrepr="?"):
         conditions = [cond.repr(argrepr) for cond in self.conditions]
+        # slow but who cares
+        conditions = "\n".join(conditions).split("\n")
         # make fake jit-debug ops to print
         for i in range(len(conditions)):
             conditions[i] = "jit_debug('%s')" % (conditions[i], )

From pypy.commits at gmail.com  Mon Aug  8 12:48:10 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 08 Aug 2016 09:48:10 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: make using objects as keys in
 dicts/in sets not read their types
Message-ID: <57a8b7ca.411d1c0a.81e64.6185@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86082:71a4e8e68f14
Date: 2016-07-07 18:33 +0200
http://bitbucket.org/pypy/pypy/changeset/71a4e8e68f14/

Log:	make using objects as keys in dicts/in sets not read their types

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1766,6 +1766,11 @@
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
         """)
 
+    def compares_by_identity(self, w_obj):
+        """ returns True if the object compares by identity (ie inherits __eq__
+        and __hash__ from object) """
+        return self.type(w_obj).compares_by_identity()
+
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -592,16 +592,16 @@
         return self.erase(None)
 
     def switch_to_correct_strategy(self, w_dict, w_key):
+        from pypy.objspace.std.intobject import W_IntObject
         if type(w_key) is self.space.StringObjectCls:
             self.switch_to_bytes_strategy(w_dict)
             return
         elif type(w_key) is self.space.UnicodeObjectCls:
             self.switch_to_unicode_strategy(w_dict)
             return
-        w_type = self.space.type(w_key)
-        if self.space.is_w(w_type, self.space.w_int):
+        elif type(w_key) is W_IntObject:
             self.switch_to_int_strategy(w_dict)
-        elif w_type.compares_by_identity():
+        if self.space.compares_by_identity(w_key):
             self.switch_to_identity_strategy(w_dict)
         else:
             self.switch_to_object_strategy(w_dict)
diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -70,8 +70,7 @@
         return self.erase(d)
 
     def is_correct_type(self, w_obj):
-        w_type = self.space.type(w_obj)
-        return w_type.compares_by_identity()
+        return self.space.compares_by_identity(w_obj)
 
     def _never_equal_to(self, w_lookup_type):
         return False
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -352,6 +352,9 @@
         from pypy.objspace.std.typeobject import _issubtype
         return _issubtype(self.terminator.w_cls, w_type)
 
+    @jit.elidable_compatible(quasi_immut_field_name_for_second_arg="version")
+    def _type_compares_by_identity(self, version):
+        return self.terminator.w_cls.compares_by_identity()
 
 class Terminator(AbstractAttribute):
     _immutable_fields_ = ['w_cls']
@@ -1172,3 +1175,10 @@
                 if version_tag is not None:
                     return map._type_issubtype(w_type)
     return space.type(w_obj).issubtype(w_type)
+
+def mapdict_compares_by_identity(space, w_obj):
+    if we_are_jitted() and w_obj.user_overridden_class:
+        map = w_obj._get_mapdict_map_no_promote()
+        if map.version is not None:
+            return map._type_compares_by_identity()
+    return space.type(w_obj).compares_by_identity()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -657,6 +657,10 @@
                     return True
         return mapdict_type_isinstance(self, w_inst, w_type)
 
+    def compares_by_identity(self, w_obj):
+        from pypy.objspace.std.mapdict import mapdict_compares_by_identity
+        return mapdict_compares_by_identity(self, w_obj)
+
     @specialize.memo()
     def _get_interplevel_cls(self, w_type):
         if not hasattr(self, "_interplevel_classes"):
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -796,7 +796,7 @@
             strategy = self.space.fromcache(BytesSetStrategy)
         elif type(w_key) is W_UnicodeObject:
             strategy = self.space.fromcache(UnicodeSetStrategy)
-        elif self.space.type(w_key).compares_by_identity():
+        elif self.space.compares_by_identity(w_key):
             strategy = self.space.fromcache(IdentitySetStrategy)
         else:
             strategy = self.space.fromcache(ObjectSetStrategy)
@@ -1399,8 +1399,7 @@
         return {}
 
     def is_correct_type(self, w_key):
-        w_type = self.space.type(w_key)
-        return w_type.compares_by_identity()
+        return self.space.compares_by_identity(w_key)
 
     def may_contain_equal_elements(self, strategy):
         #empty first, probably more likely
@@ -1645,7 +1644,7 @@
 
     # check for compares by identity
     for w_item in iterable_w:
-        if not space.type(w_item).compares_by_identity():
+        if not space.compares_by_identity(w_item):
             break
     else:
         w_set.strategy = space.fromcache(IdentitySetStrategy)
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1241,7 +1241,7 @@
         assert (Y < Y) is True
 
 
-class AppTestComparesByIdentity:
+class AppTestComparesByIdentity(jit.RandomWeAreJittedTestMixin):
 
     def setup_class(cls):
         if cls.runappdirect:
@@ -1251,6 +1251,10 @@
             return space.wrap(w_cls.compares_by_identity())
         cls.w_compares_by_identity = cls.space.wrap(interp2app(compares_by_identity))
 
+        def instance_compares_by_identity(space, w_obj):
+            return space.wrap(space.compares_by_identity(w_obj))
+        cls.w_instance_compares_by_identity = cls.space.wrap(interp2app(instance_compares_by_identity))
+
     def test_compares_by_identity(self):
         class Plain(object):
             pass
@@ -1282,6 +1286,15 @@
         assert self.compares_by_identity(TypeSubclass)
         assert not self.compares_by_identity(TypeSubclassCustomCmp)
 
+        assert self.instance_compares_by_identity(Plain())
+        assert not self.instance_compares_by_identity(CustomEq())
+        assert not self.instance_compares_by_identity(CustomCmp())
+        assert not self.instance_compares_by_identity(CustomHash())
+        assert self.instance_compares_by_identity(Plain)
+        assert self.instance_compares_by_identity(TypeSubclass('a', (object, ), {}))
+        assert not self.instance_compares_by_identity(TypeSubclassCustomCmp('a', (object, ), {}))
+
+
     def test_modify_class(self):
         class X(object):
             pass

From pypy.commits at gmail.com  Mon Aug  8 12:48:12 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 08 Aug 2016 09:48:12 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix test_dictmultiobject.py
Message-ID: <57a8b7cc.ca11c30a.76936.9bda@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86083:f1fe34175fc1
Date: 2016-08-05 18:28 +0200
http://bitbucket.org/pypy/pypy/changeset/f1fe34175fc1/

Log:	fix test_dictmultiobject.py

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -592,16 +592,15 @@
         return self.erase(None)
 
     def switch_to_correct_strategy(self, w_dict, w_key):
-        from pypy.objspace.std.intobject import W_IntObject
         if type(w_key) is self.space.StringObjectCls:
             self.switch_to_bytes_strategy(w_dict)
             return
         elif type(w_key) is self.space.UnicodeObjectCls:
             self.switch_to_unicode_strategy(w_dict)
             return
-        elif type(w_key) is W_IntObject:
+        elif type(w_key) is self.space.IntObjectCls:
             self.switch_to_int_strategy(w_dict)
-        if self.space.compares_by_identity(w_key):
+        elif self.space.compares_by_identity(w_key):
             self.switch_to_identity_strategy(w_dict)
         else:
             self.switch_to_object_strategy(w_dict)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -49,6 +49,7 @@
         self.FrameClass = frame.build_frame(self)
         self.StringObjectCls = W_BytesObject
         self.UnicodeObjectCls = W_UnicodeObject
+        self.IntObjectCls = W_IntObject
 
         # singletons
         self.w_None = W_NoneObject.w_None
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1108,11 +1108,15 @@
     w_float = float
     StringObjectCls = FakeString
     UnicodeObjectCls = FakeUnicode
+    IntObjectCls = int
     w_dict = W_DictObject
     iter = iter
     fixedview = list
     listview  = list
 
+    def compares_by_identity(self, w_obj):
+        return False # safe default
+
 class Config:
     class objspace:
         class std:

From pypy.commits at gmail.com  Mon Aug  8 12:48:14 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 08 Aug 2016 09:48:14 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: make _type_issubtype function
 actually elidable
Message-ID: <57a8b7ce.81cb1c0a.a9579.a326@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86084:8bfdd10930a7
Date: 2016-08-08 18:47 +0200
http://bitbucket.org/pypy/pypy/changeset/8bfdd10930a7/

Log:	make _type_issubtype function actually elidable

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -348,7 +348,7 @@
         return w_res
 
     @jit.elidable_compatible(quasi_immut_field_name_for_second_arg="version")
-    def _type_issubtype(self, version, w_type):
+    def _type_issubtype(self, version, w_type, w_type_version):
         from pypy.objspace.std.typeobject import _issubtype
         return _issubtype(self.terminator.w_cls, w_type)
 
@@ -1173,7 +1173,7 @@
             if map.version is not None:
                 version_tag = w_type.version_tag()
                 if version_tag is not None:
-                    return map._type_issubtype(w_type)
+                    return map._type_issubtype(w_type, version_tag)
     return space.type(w_obj).issubtype(w_type)
 
 def mapdict_compares_by_identity(space, w_obj):

From pypy.commits at gmail.com  Mon Aug  8 13:02:53 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 08 Aug 2016 10:02:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: remove setup_class in test_asyncio,
 make better use of test_gil_issue
Message-ID: <57a8bb3d.151a1c0a.593ef.6364@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86085:9faaeb88a806
Date: 2016-08-08 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/9faaeb88a806/

Log:	remove setup_class in test_asyncio, make better use of
	test_gil_issue

diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -2,16 +2,13 @@
     
     spaceconfig = dict(usemodules=["select","_socket","thread","signal","struct","_multiprocessing","array","_posixsubprocess","fcntl","unicodedata"])
     
-    def setup_class(cls):
-        cls.space.appexec([], """():
+    def test_gil_issue(self):
+        # the problem occured at await asyncio.open_connection after calling run_until_complete
+        """
         import encodings.idna
         import asyncio
         async def f():
             reader, writer = await asyncio.open_connection('example.com', 80)
         
         loop = asyncio.get_event_loop()
-        loop.run_until_complete(f())""")
-    
-    def test_gil_issue(self):
-        #needed to execute setup_call in the first place
-        assert 1==1
\ No newline at end of file
+        loop.run_until_complete(f())"""

From pypy.commits at gmail.com  Mon Aug  8 13:08:50 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 08 Aug 2016 10:08:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix test,
 realloc frees the input ptr if it is realloc() ed
Message-ID: <57a8bca2.d4e41c0a.d14b6.5566@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86086:cffad389be1e
Date: 2016-08-08 20:07 +0300
http://bitbucket.org/pypy/pypy/changeset/cffad389be1e/

Log:	fix test, realloc frees the input ptr if it is realloc() ed

diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -245,12 +245,11 @@
                  char *copy, *orig = PyObject_MALLOC(12);
                  memcpy(orig, "hello world", 12);
                  copy = PyObject_REALLOC(orig, 15);
+                 /* realloc() takes care of freeing orig, if changed */
                  if (copy == NULL)
                      Py_RETURN_NONE;
                  ret = PyString_FromStringAndSize(copy, 12);
-                 if (copy != orig)
-                     PyObject_Free(copy);
-                 PyObject_Free(orig);
+                 PyObject_Free(copy);
                  return ret;
              """)])
         x = module.realloctest()

From pypy.commits at gmail.com  Mon Aug  8 13:27:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 08 Aug 2016 10:27:49 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: We get nonsense with
 stacklets, but avoid crashing, at least
Message-ID: <57a8c115.c19d1c0a.83f5f.6786@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86087:e39187569474
Date: 2016-08-08 15:08 +0200
http://bitbucket.org/pypy/pypy/changeset/e39187569474/

Log:	We get nonsense with stacklets, but avoid crashing, at least

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -98,6 +98,9 @@
 
 def leave_code(s):
     if not we_are_translated():
+        # xxx this assertion may be false in the presence of
+        # stacklets, but let's assume we never run untranslated
+        # tests with stacklets and rvmprof
         assert vmprof_tl_stack.getraw() == s
     vmprof_tl_stack.setraw(s.c_next)
     lltype.free(s, flavor='raw')
@@ -140,5 +143,9 @@
         enter_code(unique_id)    # ignore the return value
     else:
         s = vmprof_tl_stack.getraw()
-        assert s.c_value == unique_id
-        leave_code(s)
+        #assert s.c_value == unique_id and s.c_kind == VMPROF_CODE_TAG
+        #^^^ this is false in the presence of stacklets.
+        # we get random nonsense then; let's say it's ok for now
+        # and avoid crashing.
+        if s.c_value == unique_id and s.c_kind == VMPROF_CODE_TAG:
+            leave_code(s)

From pypy.commits at gmail.com  Mon Aug  8 13:27:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 08 Aug 2016 10:27:51 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: JIT fix: make
 '_vmprof_unique_id' an immutable field; I think
Message-ID: <57a8c117.c70a1c0a.ea799.6208@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86088:e09e83478469
Date: 2016-08-08 19:07 +0200
http://bitbucket.org/pypy/pypy/changeset/e09e83478469/

Log:	JIT fix: make '_vmprof_unique_id' an immutable field; I think
	decorated_function() reads it all the time, without this

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -46,6 +46,7 @@
     def _cleanup_(self):
         self.is_enabled = False
 
+    @jit.dont_look_inside
     @specialize.argtype(1)
     def register_code(self, code, full_name_func):
         """Register the code object.  Call when a new code object is made.
@@ -87,6 +88,8 @@
         if CodeClass in self._code_classes:
             return
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
+        immut = CodeClass.__dict__.get('_immutable_fields_', [])
+        CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
         self._code_classes.add(CodeClass)
         #
         class WeakCodeObjectList(RWeakListMixin):
@@ -111,6 +114,7 @@
         prev = self._gather_all_code_objs
         self._gather_all_code_objs = gather_all_code_objs
 
+    @jit.dont_look_inside
     def enable(self, fileno, interval):
         """Enable vmprof.  Writes go to the given 'fileno'.
         The sampling interval is given by 'interval' as a number of
@@ -131,6 +135,7 @@
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
 
+    @jit.dont_look_inside
     def disable(self):
         """Disable vmprof.
         Raises VMProfError if something goes wrong.

From pypy.commits at gmail.com  Mon Aug  8 13:45:49 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 08 Aug 2016 10:45:49 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Update description of
 dictproxyobject.py
Message-ID: <57a8c54d.68adc20a.25dd3.b582@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r86089:d751064b79e4
Date: 2016-08-08 18:45 +0100
http://bitbucket.org/pypy/pypy/changeset/d751064b79e4/

Log:	Update description of dictproxyobject.py

diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -1,5 +1,8 @@
-# Read-only proxy for mappings. PyPy does not have a separate type for
-# type.__dict__, so PyDictProxy_New has to use a custom read-only mapping.
+"""
+Read-only proxy for mappings.
+
+Its main use is as the return type of cls.__dict__.
+"""
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt
@@ -18,7 +21,8 @@
                 space.isinstance_w(w_mapping, space.w_list) or
                 space.isinstance_w(w_mapping, space.w_tuple)):
             raise oefmt(space.w_TypeError,
-                        "mappingproxy() argument must be a mapping, not %T", w_mapping)
+                        "mappingproxy() argument must be a mapping, not %T",
+                        w_mapping)
         return W_DictProxyObject(w_mapping)
 
     def descr_init(self, space, __args__):

From pypy.commits at gmail.com  Mon Aug  8 13:49:08 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 08 Aug 2016 10:49:08 -0700 (PDT)
Subject: [pypy-commit] pypy mappingproxy: Close branch mappingproxy
Message-ID: <57a8c614.44ce1c0a.a84b7.6d36@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: mappingproxy
Changeset: r86090:61f6e395e2e4
Date: 2016-08-08 18:48 +0100
http://bitbucket.org/pypy/pypy/changeset/61f6e395e2e4/

Log:	Close branch mappingproxy


From pypy.commits at gmail.com  Mon Aug  8 13:49:26 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 08 Aug 2016 10:49:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merged in mappingproxy (pull request #469)
Message-ID: <57a8c626.85c11c0a.6c2fc.7372@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86091:25c8dc3d6d4c
Date: 2016-08-08 18:48 +0100
http://bitbucket.org/pypy/pypy/changeset/25c8dc3d6d4c/

Log:	Merged in mappingproxy (pull request #469)

	Fix the mappingproxy type to behave as in CPython and consolidate
	its duplicate implementations in cpyext and objspace into a single
	one. Note that app-level cls.__dict__ and C-level cls->tp_dict now
	return different objects with the former being an opaque (at app-
	level) wrapper around the latter.

diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -175,7 +175,7 @@
          "__new__"      : make_new(class_name),
          }
     pycppclass = metacpp(class_name, _drop_cycles(bases), d)
- 
+
     # cache result early so that the class methods can find the class itself
     setattr(scope, final_class_name, pycppclass)
 
@@ -192,13 +192,10 @@
     for dm_name in cppclass.get_datamember_names():
         cppdm = cppclass.get_datamember(dm_name)
 
-        # here, setattr() can not be used, because a data member can shadow one in
-        # its base class, resulting in the __set__() of its base class being called
-        # by setattr(); so, store directly on the dictionary
-        pycppclass.__dict__[dm_name] = cppdm
+        setattr(pycppclass, dm_name, cppdm)
         import cppyy
         if cppyy._is_static(cppdm):     # TODO: make this a method of cppdm
-            metacpp.__dict__[dm_name] = cppdm
+            setattr(metacpp, dm_name, cppdm)
 
     # the call to register will add back-end specific pythonizations and thus
     # needs to run first, so that the generic pythonizations can use them
@@ -413,7 +410,7 @@
         lib = cppyy._load_dictionary(name)
         _loaded_dictionaries[name] = lib
         return lib
-    
+
 def _init_pythonify():
     # cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause cppyy to be loaded
diff --git a/pypy/module/cpyext/dictproxyobject.py b/pypy/module/cpyext/dictproxyobject.py
--- a/pypy/module/cpyext/dictproxyobject.py
+++ b/pypy/module/cpyext/dictproxyobject.py
@@ -1,67 +1,7 @@
-# Read-only proxy for mappings. PyPy does not have a separate type for
-# type.__dict__, so PyDictProxy_New has to use a custom read-only mapping.
-
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.interpreter.typedef import TypeDef, interp2app
+from pypy.objspace.std.dictproxyobject import W_DictProxyObject
 from pypy.module.cpyext.api import cpython_api, build_type_checkers
 from pypy.module.cpyext.pyobject import PyObject
 
-class W_DictProxyObject(W_Root):
-    "Read-only proxy for mappings."
-
-    def __init__(self, w_mapping):
-        self.w_mapping = w_mapping
-
-    def descr_len(self, space):
-        return space.len(self.w_mapping)
-
-    def descr_getitem(self, space, w_key):
-        return space.getitem(self.w_mapping, w_key)
-
-    def descr_contains(self, space, w_key):
-        return space.contains(self.w_mapping, w_key)
-
-    def descr_iter(self, space):
-        return space.iter(self.w_mapping)
-
-    def descr_str(self, space):
-        return space.str(self.w_mapping)
-
-    def descr_repr(self, space):
-        return space.repr(self.w_mapping)
-
-    @unwrap_spec(w_default=WrappedDefault(None))
-    def get_w(self, space, w_key, w_default):
-        return space.call_method(self.w_mapping, "get", w_key, w_default)
-
-    def keys_w(self, space):
-        return space.call_method(self.w_mapping, "keys")
-
-    def values_w(self, space):
-        return space.call_method(self.w_mapping, "values")
-
-    def items_w(self, space):
-        return space.call_method(self.w_mapping, "items")
-
-    def copy_w(self, space):
-        return space.call_method(self.w_mapping, "copy")
-
-W_DictProxyObject.typedef = TypeDef(
-    'mappingproxy',
-    __len__=interp2app(W_DictProxyObject.descr_len),
-    __getitem__=interp2app(W_DictProxyObject.descr_getitem),
-    __contains__=interp2app(W_DictProxyObject.descr_contains),
-    __iter__=interp2app(W_DictProxyObject.descr_iter),
-    __str__=interp2app(W_DictProxyObject.descr_str),
-    __repr__=interp2app(W_DictProxyObject.descr_repr),
-    get=interp2app(W_DictProxyObject.get_w),
-    keys=interp2app(W_DictProxyObject.keys_w),
-    values=interp2app(W_DictProxyObject.values_w),
-    items=interp2app(W_DictProxyObject.items_w),
-    copy=interp2app(W_DictProxyObject.copy_w)
-)
-
 PyDictProxy_Check, PyDictProxy_CheckExact = build_type_checkers(
     "DictProxy", W_DictProxyObject)
 
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -277,9 +277,41 @@
                      args->ob_type->tp_dict, "copy");
                  Py_INCREF(method);
                  return method;
-             ''')])
+             '''),
+            ("get_type_dict", "METH_O",
+             '''
+                PyObject* value = args->ob_type->tp_dict;
+                if (value == NULL) value = Py_None;
+                Py_INCREF(value);
+                return value;
+             '''),
+            ])
         obj = foo.new()
         assert module.read_tp_dict(obj) == foo.fooType.copy
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
+        class A(object):
+            pass
+        obj = A()
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
+        d = module.get_type_dict(1)
+        assert type(d) is dict
+        try:
+            d["_some_attribute"] = 1
+        except TypeError:  # on PyPy, int.__dict__ is really immutable
+            pass
+        else:
+            assert int._some_attribute == 1
+            del d["_some_attribute"]
 
     def test_custom_allocation(self):
         foo = self.import_module("foo")
@@ -348,6 +380,21 @@
 
         api.Py_DecRef(ref)
 
+    def test_type_dict(self, space, api):
+        w_class = space.appexec([], """():
+            class A(object):
+                pass
+            return A
+            """)
+        ref = make_ref(space, w_class)
+
+        py_type = rffi.cast(PyTypeObjectPtr, ref)
+        w_dict = from_ref(space, py_type.c_tp_dict)
+        w_name = space.newunicode(u'a')
+        space.setitem(w_dict, w_name, space.wrap(1))
+        assert space.int_w(space.getattr(w_class, w_name)) == 1
+        space.delitem(w_dict, w_name)
+
     def test_multiple_inheritance(self, space, api):
         w_class = space.appexec([], """():
             class A(object):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -192,7 +192,7 @@
     py_methoddescr.c_d_method = w_obj.ml
 
 def classmethoddescr_realize(space, obj):
-    # XXX NOT TESTED When is this ever called? 
+    # XXX NOT TESTED When is this ever called?
     method = rffi.cast(lltype.Ptr(PyMethodDef), obj)
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
     w_obj = space.allocate_instance(W_PyCClassMethodObject, w_type)
@@ -201,7 +201,7 @@
     return w_obj
 
 def methoddescr_realize(space, obj):
-    # XXX NOT TESTED When is this ever called? 
+    # XXX NOT TESTED When is this ever called?
     method = rffi.cast(lltype.Ptr(PyMethodDef), obj)
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
     w_obj = space.allocate_instance(W_PyCMethodObject, w_type)
@@ -272,12 +272,12 @@
         if len(slot_names) == 1:
             if not getattr(pto, slot_names[0]):
                 setattr(pto, slot_names[0], slot_func_helper)
-        elif (w_type.getname(space) in ('list', 'tuple') and 
+        elif (w_type.getname(space) in ('list', 'tuple') and
               slot_names[0] == 'c_tp_as_number'):
             # XXX hack - hwo can we generalize this? The problem is method
             # names like __mul__ map to more than one slot, and we have no
             # convenient way to indicate which slots CPython have filled
-            # 
+            #
             # We need at least this special case since Numpy checks that
             # (list, tuple) do __not__ fill tp_as_number
             pass
@@ -767,8 +767,8 @@
 
     if w_obj.is_cpytype():
         Py_DecRef(space, pto.c_tp_dict)
-        w_dict = w_obj.getdict(space)
-        pto.c_tp_dict = make_ref(space, w_dict)
+    w_dict = w_obj.getdict(space)
+    pto.c_tp_dict = make_ref(space, w_dict)
 
 @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL)
 def PyType_IsSubtype(space, a, b):
diff --git a/pypy/objspace/std/classdict.py b/pypy/objspace/std/classdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/classdict.py
@@ -0,0 +1,119 @@
+from rpython.rlib import rerased
+from rpython.rlib.objectmodel import iteritems_with_hash
+
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.objspace.std.dictmultiobject import (
+    DictStrategy, create_iterator_classes)
+from pypy.objspace.std.typeobject import unwrap_cell
+
+
+class ClassDictStrategy(DictStrategy):
+    """Exposes a W_TypeObject.dict_w at app-level.
+
+    Uses getdictvalue() and setdictvalue() to access items.
+    """
+    erase, unerase = rerased.new_erasing_pair("dictproxy")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        w_lookup_type = space.type(w_key)
+        if space.issubtype_w(w_lookup_type, space.w_unicode):
+            return self.getitem_str(w_dict, space.str_w(w_key))
+        else:
+            return None
+
+    def getitem_str(self, w_dict, key):
+        return self.unerase(w_dict.dstorage).getdictvalue(self.space, key)
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if space.is_w(space.type(w_key), space.w_unicode):
+            self.setitem_str(w_dict, self.space.str_w(w_key), w_value)
+        else:
+            raise oefmt(space.w_TypeError,
+                        "cannot add non-string keys to dict of a type")
+
+    def setitem_str(self, w_dict, key, w_value):
+        w_type = self.unerase(w_dict.dstorage)
+        try:
+            w_type.setdictvalue(self.space, key, w_value)
+        except OperationError as e:
+            if not e.match(self.space, self.space.w_TypeError):
+                raise
+            if not w_type.is_cpytype():
+                raise
+            # Allow cpyext to write to type->tp_dict even in the case
+            # of a builtin type.
+            # Like CPython, we assume that this is only done early
+            # after the type is created, and we don't invalidate any
+            # cache.  User code shoud call PyType_Modified().
+            w_type.dict_w[key] = w_value
+
+    def setdefault(self, w_dict, w_key, w_default):
+        w_result = self.getitem(w_dict, w_key)
+        if w_result is not None:
+            return w_result
+        self.setitem(w_dict, w_key, w_default)
+        return w_default
+
+    def delitem(self, w_dict, w_key):
+        space = self.space
+        w_key_type = space.type(w_key)
+        if space.is_w(w_key_type, space.w_unicode):
+            key = self.space.str_w(w_key)
+            if not self.unerase(w_dict.dstorage).deldictvalue(space, key):
+                raise KeyError
+        else:
+            raise KeyError
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage).dict_w)
+
+    def w_keys(self, w_dict):
+        space = self.space
+        w_type = self.unerase(w_dict.dstorage)
+        return space.newlist([_wrapkey(space, key)
+                              for key in w_type.dict_w.iterkeys()])
+
+    def values(self, w_dict):
+        return [unwrap_cell(self.space, w_value) for w_value in
+                self.unerase(w_dict.dstorage).dict_w.itervalues()]
+
+    def items(self, w_dict):
+        space = self.space
+        w_type = self.unerase(w_dict.dstorage)
+        return [space.newtuple([_wrapkey(space, key),
+                                unwrap_cell(space, w_value)])
+                for (key, w_value) in w_type.dict_w.iteritems()]
+
+    def clear(self, w_dict):
+        space = self.space
+        w_type = self.unerase(w_dict.dstorage)
+        if not w_type.is_heaptype():
+            raise oefmt(space.w_TypeError,
+                        "can't clear dictionary of type '%N'", w_type)
+        w_type.dict_w.clear()
+        w_type.mutated(None)
+
+    def getiterkeys(self, w_dict):
+        return self.unerase(w_dict.dstorage).dict_w.iterkeys()
+
+    def getitervalues(self, w_dict):
+        return self.unerase(w_dict.dstorage).dict_w.itervalues()
+
+    def getiteritems_with_hash(self, w_dict):
+        return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w)
+
+    def wrapkey(space, key):
+        return _wrapkey(space, key)
+
+    def wrapvalue(space, value):
+        return unwrap_cell(space, value)
+
+def _wrapkey(space, key):
+    # keys are utf-8 encoded identifiers from type's dict_w
+    return space.wrap(key.decode('utf-8'))
+
+create_iterator_classes(ClassDictStrategy)
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -1,212 +1,95 @@
-from rpython.rlib import rerased
-from rpython.rlib.objectmodel import iteritems_with_hash
+"""
+Read-only proxy for mappings.
 
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.typedef import TypeDef
-from pypy.objspace.std.dictmultiobject import (
-    DictStrategy, W_DictObject, create_iterator_classes)
-from pypy.objspace.std.typeobject import unwrap_cell
+Its main use is as the return type of cls.__dict__.
+"""
 
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
+from pypy.interpreter.typedef import TypeDef, interp2app
 
-class W_DictProxyObject(W_DictObject):
+class W_DictProxyObject(W_Root):
+    "Read-only proxy for mappings."
+
+    def __init__(self, w_mapping):
+        self.w_mapping = w_mapping
+
     @staticmethod
     def descr_new(space, w_type, w_mapping):
         if (not space.lookup(w_mapping, "__getitem__") or
-            space.isinstance_w(w_mapping, space.w_list) or
-            space.isinstance_w(w_mapping, space.w_tuple)):
+                space.isinstance_w(w_mapping, space.w_list) or
+                space.isinstance_w(w_mapping, space.w_tuple)):
             raise oefmt(space.w_TypeError,
-                        "mappingproxy() argument must be a mapping, not %T", w_mapping)
-        strategy = space.fromcache(MappingProxyStrategy)
-        storage = strategy.erase(w_mapping)
-        w_obj = space.allocate_instance(W_DictProxyObject, w_type)
-        W_DictProxyObject.__init__(w_obj, space, strategy, storage)
-        return w_obj
+                        "mappingproxy() argument must be a mapping, not %T",
+                        w_mapping)
+        return W_DictProxyObject(w_mapping)
 
     def descr_init(self, space, __args__):
         pass
 
+    def descr_len(self, space):
+        return space.len(self.w_mapping)
+
+    def descr_getitem(self, space, w_key):
+        return space.getitem(self.w_mapping, w_key)
+
+    def descr_contains(self, space, w_key):
+        return space.contains(self.w_mapping, w_key)
+
+    def descr_iter(self, space):
+        return space.iter(self.w_mapping)
+
+    def descr_str(self, space):
+        return space.str(self.w_mapping)
+
     def descr_repr(self, space):
-        return space.wrap(u"mappingproxy(%s)" % (
-            space.unicode_w(W_DictObject.descr_repr(self, space))))
+        return space.newunicode(u"mappingproxy(%s)" %
+                                (space.unicode_w(space.repr(self.w_mapping)),))
+
+    @unwrap_spec(w_default=WrappedDefault(None))
+    def get_w(self, space, w_key, w_default):
+        return space.call_method(self.w_mapping, "get", w_key, w_default)
+
+    def keys_w(self, space):
+        return space.call_method(self.w_mapping, "keys")
+
+    def values_w(self, space):
+        return space.call_method(self.w_mapping, "values")
+
+    def items_w(self, space):
+        return space.call_method(self.w_mapping, "items")
+
+    def copy_w(self, space):
+        return space.call_method(self.w_mapping, "copy")
+
+cmp_methods = {}
+def make_cmp_method(op):
+    def descr_op(self, space, w_other):
+        return getattr(space, op)(self.w_mapping, w_other)
+    descr_name = 'descr_' + op
+    descr_op.__name__ = descr_name
+    setattr(W_DictProxyObject, descr_name, descr_op)
+    cmp_methods['__%s__' % op] = interp2app(getattr(W_DictProxyObject, descr_name))
+
+for op in ['eq', 'ne', 'gt', 'ge', 'lt', 'le']:
+    make_cmp_method(op)
+
 
 W_DictProxyObject.typedef = TypeDef(
-    "mappingproxy", W_DictObject.typedef,
-    __new__ = interp2app(W_DictProxyObject.descr_new),
-    __init__ = interp2app(W_DictProxyObject.descr_init),
-    __repr__ = interp2app(W_DictProxyObject.descr_repr),
+    'mappingproxy',
+    __new__=interp2app(W_DictProxyObject.descr_new),
+    __init__=interp2app(W_DictProxyObject.descr_init),
+    __len__=interp2app(W_DictProxyObject.descr_len),
+    __getitem__=interp2app(W_DictProxyObject.descr_getitem),
+    __contains__=interp2app(W_DictProxyObject.descr_contains),
+    __iter__=interp2app(W_DictProxyObject.descr_iter),
+    __str__=interp2app(W_DictProxyObject.descr_str),
+    __repr__=interp2app(W_DictProxyObject.descr_repr),
+    get=interp2app(W_DictProxyObject.get_w),
+    keys=interp2app(W_DictProxyObject.keys_w),
+    values=interp2app(W_DictProxyObject.values_w),
+    items=interp2app(W_DictProxyObject.items_w),
+    copy=interp2app(W_DictProxyObject.copy_w),
+    **cmp_methods
 )
-
-
-class DictProxyStrategy(DictStrategy):
-    """Exposes a W_TypeObject.dict_w at app-level.
-
-    Uses getdictvalue() and setdictvalue() to access items.
-    """
-    erase, unerase = rerased.new_erasing_pair("dictproxy")
-    erase = staticmethod(erase)
-    unerase = staticmethod(unerase)
-
-    def getitem(self, w_dict, w_key):
-        space = self.space
-        w_lookup_type = space.type(w_key)
-        if space.issubtype_w(w_lookup_type, space.w_unicode):
-            return self.getitem_str(w_dict, space.str_w(w_key))
-        else:
-            return None
-
-    def getitem_str(self, w_dict, key):
-        return self.unerase(w_dict.dstorage).getdictvalue(self.space, key)
-
-    def setitem(self, w_dict, w_key, w_value):
-        space = self.space
-        if space.is_w(space.type(w_key), space.w_unicode):
-            self.setitem_str(w_dict, self.space.str_w(w_key), w_value)
-        else:
-            raise oefmt(space.w_TypeError,
-                        "cannot add non-string keys to dict of a type")
-
-    def setitem_str(self, w_dict, key, w_value):
-        w_type = self.unerase(w_dict.dstorage)
-        try:
-            w_type.setdictvalue(self.space, key, w_value)
-        except OperationError as e:
-            if not e.match(self.space, self.space.w_TypeError):
-                raise
-            if not w_type.is_cpytype():
-                raise
-            # Allow cpyext to write to type->tp_dict even in the case
-            # of a builtin type.
-            # Like CPython, we assume that this is only done early
-            # after the type is created, and we don't invalidate any
-            # cache.  User code shoud call PyType_Modified().
-            w_type.dict_w[key] = w_value
-
-    def setdefault(self, w_dict, w_key, w_default):
-        w_result = self.getitem(w_dict, w_key)
-        if w_result is not None:
-            return w_result
-        self.setitem(w_dict, w_key, w_default)
-        return w_default
-
-    def delitem(self, w_dict, w_key):
-        space = self.space
-        w_key_type = space.type(w_key)
-        if space.is_w(w_key_type, space.w_unicode):
-            key = self.space.str_w(w_key)
-            if not self.unerase(w_dict.dstorage).deldictvalue(space, key):
-                raise KeyError
-        else:
-            raise KeyError
-
-    def length(self, w_dict):
-        return len(self.unerase(w_dict.dstorage).dict_w)
-
-    def w_keys(self, w_dict):
-        space = self.space
-        w_type = self.unerase(w_dict.dstorage)
-        return space.newlist([_wrapkey(space, key)
-                              for key in w_type.dict_w.iterkeys()])
-
-    def values(self, w_dict):
-        return [unwrap_cell(self.space, w_value) for w_value in self.unerase(w_dict.dstorage).dict_w.itervalues()]
-
-    def items(self, w_dict):
-        space = self.space
-        w_type = self.unerase(w_dict.dstorage)
-        return [space.newtuple([_wrapkey(space, key),
-                                unwrap_cell(space, w_value)])
-                for (key, w_value) in w_type.dict_w.iteritems()]
-
-    def clear(self, w_dict):
-        space = self.space
-        w_type = self.unerase(w_dict.dstorage)
-        if not w_type.is_heaptype():
-            raise oefmt(space.w_TypeError,
-                        "can't clear dictionary of type '%N'", w_type)
-        w_type.dict_w.clear()
-        w_type.mutated(None)
-
-    def getiterkeys(self, w_dict):
-        return self.unerase(w_dict.dstorage).dict_w.iterkeys()
-    def getitervalues(self, w_dict):
-        return self.unerase(w_dict.dstorage).dict_w.itervalues()
-    def getiteritems_with_hash(self, w_dict):
-        return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w)
-    def wrapkey(space, key):
-        return _wrapkey(space, key)
-    def wrapvalue(space, value):
-        return unwrap_cell(space, value)
-
-def _wrapkey(space, key):
-    # keys are utf-8 encoded identifiers from type's dict_w
-    return space.wrap(key.decode('utf-8'))
-
-create_iterator_classes(DictProxyStrategy)
-
-
-class MappingProxyStrategy(DictStrategy):
-    """Wraps an applevel mapping in a read-only dictionary."""
-    erase, unerase = rerased.new_erasing_pair("mappingproxy")
-    erase = staticmethod(erase)
-    unerase = staticmethod(unerase)
-
-    def getitem(self, w_dict, w_key):
-        try:
-            return self.space.getitem(self.unerase(w_dict.dstorage), w_key)
-        except OperationError as e:
-            if not e.match(self.space, self.space.w_KeyError):
-                raise
-            return None
-
-    def setitem(self, w_dict, w_key, w_value):
-        raise oefmt(self.space.w_TypeError,
-                    "'%T' object does not support item assignment", w_dict)
-
-    def delitem(self, w_dict, w_key):
-        raise oefmt(self.space.w_TypeError,
-                    "'%T' object does not support item deletion", w_dict)
-
-    def length(self, w_dict):
-        return self.space.len_w(self.unerase(w_dict.dstorage))
-
-    def getiterkeys(self, w_dict):
-        return self.space.iter(
-            self.space.call_method(self.unerase(w_dict.dstorage), "keys"))
-
-    def getitervalues(self, w_dict):
-        return self.space.iter(
-            self.space.call_method(self.unerase(w_dict.dstorage), "values"))
-
-    def getiteritems_with_hash(self, w_dict):
-        return self.space.iter(
-            self.space.call_method(self.unerase(w_dict.dstorage), "items"))
-
-    @staticmethod
-    def override_next_key(iterkeys):
-        w_keys = iterkeys.iterator
-        return iterkeys.space.next(w_keys)
-
-    @staticmethod
-    def override_next_value(itervalues):
-        w_values = itervalues.iterator
-        return itervalues.space.next(w_values)
-
-    @staticmethod
-    def override_next_item(iteritems):
-        w_items = iteritems.iterator
-        w_item = iteritems.space.next(w_items)
-        w_key, w_value = iteritems.space.unpackiterable(w_item, 2)
-        return w_key, w_value
-
-    def clear(self, w_dict):
-        raise oefmt(self.space.w_AttributeError, "clear")
-
-    def copy(self, w_dict):
-        return self.space.call_method(self.unerase(w_dict.dstorage), "copy")
-
-create_iterator_classes(
-    MappingProxyStrategy,
-    override_next_key=MappingProxyStrategy.override_next_key,
-    override_next_value=MappingProxyStrategy.override_next_value,
-    override_next_item=MappingProxyStrategy.override_next_item)
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -9,42 +9,20 @@
         assert 'a' in NotEmpty.__dict__
         assert 'a' in NotEmpty.__dict__.keys()
         assert 'b' not in NotEmpty.__dict__
-        NotEmpty.__dict__['b'] = 4
-        assert NotEmpty.b == 4
-        del NotEmpty.__dict__['b']
         assert NotEmpty.__dict__.get("b") is None
+        raises(TypeError, "NotEmpty.__dict__['b'] = 4")
         raises(TypeError, 'NotEmpty.__dict__[15] = "y"')
-        raises(KeyError, 'del NotEmpty.__dict__[15]')
+        raises(TypeError, 'del NotEmpty.__dict__[15]')
 
-        assert NotEmpty.__dict__.setdefault("string", 1) == 1
-        assert NotEmpty.__dict__.setdefault("string", 2) == 1
-        assert NotEmpty.string == 1
-        raises(TypeError, 'NotEmpty.__dict__.setdefault(15, 1)')
-
-    def test_dictproxy_popitem(self):
-        class A(object):
-            a = 42
-        seen = 0
-        try:
-            while True:
-                key, value = A.__dict__.popitem()
-                if key == 'a':
-                    assert value == 42
-                    seen += 1
-        except KeyError:
-            pass
-        assert seen == 1
+        raises(AttributeError, 'NotEmpty.__dict__.setdefault')
 
     def test_dictproxy_getitem(self):
         class NotEmpty(object):
             a = 1
         assert 'a' in NotEmpty.__dict__
-        class substr(str): pass
+        class substr(str):
+            pass
         assert substr('a') in NotEmpty.__dict__
-        # the following are only for py2
-        ## assert u'a' in NotEmpty.__dict__
-        ## assert NotEmpty.__dict__[u'a'] == 1
-        ## assert u'\xe9' not in NotEmpty.__dict__
 
     def test_dictproxyeq(self):
         class a(object):
@@ -63,9 +41,9 @@
         class a(object):
             pass
         s1 = repr(a.__dict__)
+        assert s1.startswith('mappingproxy({') and s1.endswith('})')
         s2 = str(a.__dict__)
-        assert s1 == s2
-        assert s1.startswith('mappingproxy({') and s1.endswith('})')
+        assert s1 == 'mappingproxy(%s)' % s2
 
     def test_immutable_dict_on_builtin_type(self):
         raises(TypeError, "int.__dict__['a'] = 1")
@@ -100,4 +78,3 @@
 
 class AppTestUserObjectMethodCache(AppTestUserObject):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
-
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -970,7 +970,6 @@
         raises(TypeError, setattr, list, 'foobar', 42)
         raises(TypeError, delattr, dict, 'keys')
         raises(TypeError, 'int.__dict__["a"] = 1')
-        raises(TypeError, 'int.__dict__.clear()')
 
     def test_nontype_in_mro(self):
         class OldStyle:
@@ -1028,10 +1027,9 @@
             pass
 
         a = A()
+        d = A.__dict__
         A.x = 1
-        assert A.__dict__["x"] == 1
-        A.__dict__['x'] = 5
-        assert A.x == 5
+        assert d["x"] == 1
 
     def test_we_already_got_one_1(self):
         # Issue #2079: highly obscure: CPython complains if we say
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -4,8 +4,8 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import (
     Function, StaticMethod, ClassMethod, FunctionWithFixedCode)
-from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\
-     descr_get_dict, dict_descr, Member, TypeDef
+from pypy.interpreter.typedef import (
+    weakref_descr, GetSetProperty, dict_descr, Member, TypeDef)
 from pypy.interpreter.astcompiler.misc import mangle
 from pypy.module.__builtin__ import abstractinst
 
@@ -344,7 +344,7 @@
     def deldictvalue(self, space, key):
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        if not self.is_heaptype():
+        if not (self.is_heaptype() or self.is_cpytype()):
             raise oefmt(space.w_TypeError,
                         "can't delete attributes on type object '%N'", self)
         try:
@@ -483,14 +483,14 @@
                 self.getdictvalue(self.space, attr)
             del self.lazyloaders
 
-    def getdict(self, space): # returning a dict-proxy!
-        from pypy.objspace.std.dictproxyobject import DictProxyStrategy
-        from pypy.objspace.std.dictproxyobject import W_DictProxyObject
+    def getdict(self, space):
+        from pypy.objspace.std.classdict import ClassDictStrategy
+        from pypy.objspace.std.dictmultiobject import W_DictObject
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        strategy = space.fromcache(DictProxyStrategy)
+        strategy = space.fromcache(ClassDictStrategy)
         storage = strategy.erase(self)
-        return W_DictProxyObject(space, strategy, storage)
+        return W_DictObject(space, strategy, storage)
 
     def is_heaptype(self):
         return self.flag_heaptype
@@ -929,6 +929,13 @@
     return space.newbool(
         abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj))
 
+def type_get_dict(space, w_cls):
+    from pypy.objspace.std.dictproxyobject import W_DictProxyObject
+    w_dict = w_cls.getdict(space)
+    if w_dict is None:
+        return space.w_None
+    return W_DictProxyObject(w_dict)
+
 W_TypeObject.typedef = TypeDef("type",
     __new__ = gateway.interp2app(descr__new__),
     __name__ = GetSetProperty(descr_get__name__, descr_set__name__),
@@ -936,7 +943,7 @@
     __bases__ = GetSetProperty(descr_get__bases__, descr_set__bases__),
     __base__ = GetSetProperty(descr__base),
     __mro__ = GetSetProperty(descr_get__mro__),
-    __dict__ = GetSetProperty(descr_get_dict),
+    __dict__=GetSetProperty(type_get_dict),
     __doc__ = GetSetProperty(descr__doc, descr_set__doc),
     __dir__ = gateway.interp2app(descr__dir),
     mro = gateway.interp2app(descr_mro),

From pypy.commits at gmail.com  Mon Aug  8 13:56:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 08 Aug 2016 10:56:21 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: Tentative proper fix for
 stacklet+vmprof
Message-ID: <57a8c7c5.68adc20a.25dd3.b8a8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86092:68cb0f468f13
Date: 2016-08-08 19:55 +0200
http://bitbucket.org/pypy/pypy/changeset/68cb0f468f13/

Log:	Tentative proper fix for stacklet+vmprof

diff --git a/rpython/rlib/rstacklet.py b/rpython/rlib/rstacklet.py
--- a/rpython/rlib/rstacklet.py
+++ b/rpython/rlib/rstacklet.py
@@ -3,6 +3,7 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import fetch_translated_config
 from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rlib.rvmprof import cintf
 
 DEBUG = False
 
@@ -24,7 +25,12 @@
     def new(self, callback, arg=llmemory.NULL):
         if DEBUG:
             callback = _debug_wrapper(callback)
-        h = self._gcrootfinder.new(self, callback, arg)
+        x = cintf.save_rvmprof_stack()
+        try:
+            cintf.empty_rvmprof_stack()
+            h = self._gcrootfinder.new(self, callback, arg)
+        finally:
+            cintf.restore_rvmprof_stack(x)
         if DEBUG:
             debug.add(h)
         return h
@@ -34,7 +40,11 @@
     def switch(self, stacklet):
         if DEBUG:
             debug.remove(stacklet)
-        h = self._gcrootfinder.switch(stacklet)
+        x = cintf.save_rvmprof_stack()
+        try:
+            h = self._gcrootfinder.switch(stacklet)
+        finally:
+            cintf.restore_rvmprof_stack(x)
         if DEBUG:
             debug.add(h)
         return h
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -98,9 +98,6 @@
 
 def leave_code(s):
     if not we_are_translated():
-        # xxx this assertion may be false in the presence of
-        # stacklets, but let's assume we never run untranslated
-        # tests with stacklets and rvmprof
         assert vmprof_tl_stack.getraw() == s
     vmprof_tl_stack.setraw(s.c_next)
     lltype.free(s, flavor='raw')
@@ -143,9 +140,17 @@
         enter_code(unique_id)    # ignore the return value
     else:
         s = vmprof_tl_stack.getraw()
-        #assert s.c_value == unique_id and s.c_kind == VMPROF_CODE_TAG
-        #^^^ this is false in the presence of stacklets.
-        # we get random nonsense then; let's say it's ok for now
-        # and avoid crashing.
-        if s.c_value == unique_id and s.c_kind == VMPROF_CODE_TAG:
-            leave_code(s)
+        assert s.c_value == unique_id and s.c_kind == VMPROF_CODE_TAG
+        leave_code(s)
+
+#
+# stacklet support
+
+def save_rvmprof_stack():
+    return vmprof_tl_stack.get_or_make_raw()
+
+def empty_rvmprof_stack():
+    vmprof_tl_stack.setraw(lltype.nullptr(VMPROFSTACK))
+
+def restore_rvmprof_stack(x):
+    vmprof_tl_stack.setraw(x)

From pypy.commits at gmail.com  Tue Aug  9 03:42:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 00:42:54 -0700 (PDT)
Subject: [pypy-commit] pypy improve-vmprof-testing: close branch,
 ready to merge
Message-ID: <57a9897e.c62f1c0a.88ea8.92f3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: improve-vmprof-testing
Changeset: r86093:36f80e5d1a22
Date: 2016-08-09 09:39 +0200
http://bitbucket.org/pypy/pypy/changeset/36f80e5d1a22/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Tue Aug  9 03:42:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 00:42:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge improve-vmprof-testing
Message-ID: <57a98980.469d1c0a.f359e.9445@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86094:7b02cc6e8bfd
Date: 2016-08-09 09:42 +0200
http://bitbucket.org/pypy/pypy/changeset/7b02cc6e8bfd/

Log:	hg merge improve-vmprof-testing

	Improved vmprof support: now tries hard to not miss any Python-level
	frame in the captured stacks, even if there is the metainterp or
	blackhole interp involved.

	Also fix the stacklet (greenlet) support.

	Contains some sanity checks that will fail an assertion, even when
	running without vmprof enabled.

diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -7,64 +7,152 @@
 from rpython.jit.backend.x86.arch import WORD
 from rpython.jit.codewriter.policy import JitPolicy
 
+
 class BaseRVMProfTest(object):
-    def test_one(self):
-        py.test.skip("needs thread-locals in the JIT, which is only available "
-                     "after translation")
+
+    def setup_method(self, meth):
         visited = []
 
         def helper():
+            trace = []
             stack = cintf.vmprof_tl_stack.getraw()
-            print stack
-            if stack:
-                # not during tracing
-                visited.append(stack.c_value)
-            else:
-                visited.append(0)
+            while stack:
+                trace.append((stack.c_kind, stack.c_value))
+                stack = stack.c_next
+            visited.append(trace)
 
         llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
 
-        driver = jit.JitDriver(greens=['code'], reds='auto')
+        class CodeObj(object):
+            def __init__(self, name):
+                self.name = name
 
-        class CodeObj(object):
-            pass
-
-        def get_code_fn(code, arg):
+        def get_code_fn(codes, code, arg, c):
             return code
 
         def get_name(code):
             return "foo"
 
+        _get_vmprof().use_weaklist = False
         register_code_object_class(CodeObj, get_name)
 
-        @vmprof_execute_code("main", get_code_fn)
-        def f(code, n):
+        self.misc = visited, llfn, CodeObj, get_code_fn, get_name
+
+
+    def teardown_method(self, meth):
+        del _get_vmprof().use_weaklist
+
+
+    def test_simple(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
             i = 0
             while i < n:
-                driver.jit_merge_point(code=code)
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if code.name == "main":
+                    c = f(codes, codes[1], 1, c)
+                else:
+                    llfn()
+                    c -= 1
                 i += 1
-                llfn()
+            return c
 
         def main(n):
-            cintf.vmprof_tl_stack.setraw(null)   # make it empty
-            vmprof = _get_vmprof()
-            code = CodeObj()
-            register_code(code, get_name)
-            return f(code, n)
-
-        class Hooks(jit.JitHookInterface):
-            def after_compile(self, debug_info):
-                self.raw_start = debug_info.asminfo.rawstart
-
-        hooks = Hooks()
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            return f(codes, codes[0], n, 8)
 
         null = lltype.nullptr(cintf.VMPROFSTACK)
-        self.meta_interp(main, [10], policy=JitPolicy(hooks))
-        print visited
-        #v = set(visited)
-        #assert 0 in v
-        #v.remove(0)
-        #assert len(v) == 1
-        #assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
-        #assert cintf.vmprof_tl_stack.getraw() == null
-        # ^^^ make sure we didn't leave anything dangling
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+
+
+    def test_leaving_with_exception(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        class MyExc(Exception):
+            def __init__(self, c):
+                self.c = c
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
+            i = 0
+            while i < n:
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if code.name == "main":
+                    try:
+                        f(codes, codes[1], 1, c)
+                    except MyExc as e:
+                        c = e.c
+                else:
+                    llfn()
+                    c -= 1
+                i += 1
+            raise MyExc(c)
+
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            try:
+                f(codes, codes[0], n, 8)
+            except MyExc as e:
+                return e.c
+
+        null = lltype.nullptr(cintf.VMPROFSTACK)
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+
+
+    def test_leaving_with_exception_in_blackhole(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        class MyExc(Exception):
+            def __init__(self, c):
+                self.c = c
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
+            i = 0
+            while True:
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if i >= n:
+                    break
+                i += 1
+                if code.name == "main":
+                    try:
+                        f(codes, codes[1], 1, c)
+                    except MyExc as e:
+                        c = e.c
+                    driver.can_enter_jit(code=code, c=c, i=i, codes=codes, n=n)
+                else:
+                    llfn()
+                    c -= 1
+            if c & 1:      # a failing guard
+                pass
+            raise MyExc(c)
+
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            try:
+                f(codes, codes[0], n, 8)
+            except MyExc as e:
+                return e.c
+
+        null = lltype.nullptr(cintf.VMPROFSTACK)
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -452,6 +452,8 @@
             prepare = self._handle_math_sqrt_call
         elif oopspec_name.startswith('rgc.'):
             prepare = self._handle_rgc_call
+        elif oopspec_name.startswith('rvmprof.'):
+            prepare = self._handle_rvmprof_call
         elif oopspec_name.endswith('dict.lookup'):
             # also ordereddict.lookup
             prepare = self._handle_dict_lookup_call
@@ -2079,6 +2081,32 @@
         else:
             raise NotImplementedError(oopspec_name)
 
+    def _handle_rvmprof_call(self, op, oopspec_name, args):
+        if oopspec_name != 'rvmprof.jitted':
+            raise NotImplementedError(oopspec_name)
+        c_entering = Constant(0, lltype.Signed)
+        c_leaving  = Constant(1, lltype.Signed)
+        v_uniqueid = args[0]
+        op1 = SpaceOperation('rvmprof_code', [c_entering, v_uniqueid], None)
+        op2 = SpaceOperation('rvmprof_code', [c_leaving, v_uniqueid], None)
+        #
+        # fish fish inside the oopspec's graph for the ll_func pointer
+        block = op.args[0].value._obj.graph.startblock
+        while True:
+            assert len(block.exits) == 1
+            nextblock = block.exits[0].target
+            if nextblock.operations == ():
+                break
+            block = nextblock
+        last_op = block.operations[-1]
+        assert last_op.opname == 'direct_call'
+        c_ll_func = last_op.args[0]
+        #
+        args = [c_ll_func] + op.args[2:]
+        ops = self.rewrite_op_direct_call(SpaceOperation('direct_call',
+                                                         args, op.result))
+        return [op1] + ops + [op2]
+
     def rewrite_op_ll_read_timestamp(self, op):
         op1 = self.prepare_builtin_call(op, "ll_read_timestamp", [])
         return self.handle_residual_call(op1,
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -14,7 +14,7 @@
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from rpython.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rlib import jit
+from rpython.rlib import jit, debug
 
 
 class FakeRegAlloc:
@@ -140,7 +140,6 @@
 
     def encoding_test(self, func, args, expected,
                       transform=False, liveness=False, cc=None, jd=None):
-        
         graphs = self.make_graphs(func, args)
         #graphs[0].show()
         if transform:
@@ -1112,6 +1111,31 @@
         assert str(e.value).startswith("A virtualizable array is passed aroun")
         assert "<Descr>" in str(e.value)
 
+    def test_rvmprof_code(self):
+        from rpython.rlib.rvmprof import cintf
+        class MyFakeCallControl(FakeCallControl):
+            def guess_call_kind(self, op):
+                if 'jitted' in repr(op):
+                    return 'builtin'
+                return 'residual'
+        class X:
+            pass
+        def g(x, y):
+            debug.debug_print("foo")
+            return X()
+        @jit.oopspec("rvmprof.jitted(unique_id)")
+        def decorated_jitted_function(unique_id, *args):
+            return g(*args)
+        def f(id, x, y):
+            return decorated_jitted_function(id, x, y)
+        self.encoding_test(f, [42, 56, 74], """
+            rvmprof_code $0, %i0
+            residual_call_ir_r $<* fn g>, I[%i1, %i2], R[], <Descr> -> %r0
+            -live-
+            rvmprof_code $1, %i0
+            ref_return %r0
+        """, transform=True, cc=MyFakeCallControl())
+
 
 def check_force_cast(FROM, TO, operations, value):
     """Check that the test is correctly written..."""
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -64,6 +64,7 @@
             assert self._insns[value] is None
             self._insns[value] = key
         self.op_catch_exception = insns.get('catch_exception/L', -1)
+        self.op_rvmprof_code = insns.get('rvmprof_code/ii', -1)
         #
         all_funcs = []
         for key in self._insns:
@@ -270,6 +271,7 @@
         self.dispatch_loop      = builder.dispatch_loop
         self.descrs             = builder.descrs
         self.op_catch_exception = builder.op_catch_exception
+        self.op_rvmprof_code    = builder.op_rvmprof_code
         self.count_interpreter  = count_interpreter
         #
         if we_are_translated():
@@ -373,9 +375,32 @@
                 target = ord(code[position+1]) | (ord(code[position+2])<<8)
                 self.position = target
                 return
+            if opcode == self.op_rvmprof_code:
+                # call the 'jit_rvmprof_code(1)' for rvmprof, but then
+                # continue popping frames.  Decode the 'rvmprof_code' insn
+                # manually here.
+                from rpython.rlib.rvmprof import cintf
+                arg1 = self.registers_i[ord(code[position + 1])]
+                arg2 = self.registers_i[ord(code[position + 2])]
+                assert arg1 == 1
+                cintf.jit_rvmprof_code(arg1, arg2)
         # no 'catch_exception' insn follows: just reraise
         reraise(e)
 
+    def handle_rvmprof_enter(self):
+        code = self.jitcode.code
+        position = self.position
+        opcode = ord(code[position])
+        if opcode == self.op_rvmprof_code:
+            arg1 = self.registers_i[ord(code[position + 1])]
+            arg2 = self.registers_i[ord(code[position + 2])]
+            if arg1 == 1:
+                # we are resuming at a position that will do a
+                # jit_rvmprof_code(1), when really executed.  That's a
+                # hint for the need for a jit_rvmprof_code(0).
+                from rpython.rlib.rvmprof import cintf
+                cintf.jit_rvmprof_code(0, arg2)
+
     def copy_constants(self, registers, constants):
         """Copy jitcode.constants[0] to registers[255],
                 jitcode.constants[1] to registers[254],
@@ -1501,6 +1526,11 @@
     def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
 
+    @arguments("i", "i")
+    def bhimpl_rvmprof_code(leaving, unique_id):
+        from rpython.rlib.rvmprof import cintf
+        cintf.jit_rvmprof_code(leaving, unique_id)
+
     # ----------
     # helpers to resume running in blackhole mode when a guard failed
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1453,6 +1453,25 @@
             metainterp.history.record(rop.VIRTUAL_REF_FINISH,
                                       [vrefbox, nullbox], None)
 
+    @arguments("int", "box")
+    def opimpl_rvmprof_code(self, leaving, box_unique_id):
+        from rpython.rlib.rvmprof import cintf
+        cintf.jit_rvmprof_code(leaving, box_unique_id.getint())
+
+    def handle_rvmprof_enter_on_resume(self):
+        code = self.bytecode
+        position = self.pc
+        opcode = ord(code[position])
+        if opcode == self.metainterp.staticdata.op_rvmprof_code:
+            arg1 = self.registers_i[ord(code[position + 1])].getint()
+            arg2 = self.registers_i[ord(code[position + 2])].getint()
+            if arg1 == 1:
+                # we are resuming at a position that will do a
+                # jit_rvmprof_code(1), when really executed.  That's a
+                # hint for the need for a jit_rvmprof_code(0).
+                from rpython.rlib.rvmprof import cintf
+                cintf.jit_rvmprof_code(0, arg2)
+
     # ------------------------------
 
     def setup_call(self, argboxes):
@@ -1804,6 +1823,7 @@
             opimpl = _get_opimpl_method(name, argcodes)
             self.opcode_implementations[value] = opimpl
         self.op_catch_exception = insns.get('catch_exception/L', -1)
+        self.op_rvmprof_code = insns.get('rvmprof_code/ii', -1)
 
     def setup_descrs(self, descrs):
         self.opcode_descrs = descrs
@@ -2071,6 +2091,15 @@
                     target = ord(code[position+1]) | (ord(code[position+2])<<8)
                     frame.pc = target
                     raise ChangeFrame
+                if opcode == self.staticdata.op_rvmprof_code:
+                    # call the 'jit_rvmprof_code(1)' for rvmprof, but then
+                    # continue popping frames.  Decode the 'rvmprof_code' insn
+                    # manually here.
+                    from rpython.rlib.rvmprof import cintf
+                    arg1 = frame.registers_i[ord(code[position + 1])].getint()
+                    arg2 = frame.registers_i[ord(code[position + 2])].getint()
+                    assert arg1 == 1
+                    cintf.jit_rvmprof_code(arg1, arg2)
             self.popframe()
         try:
             self.compile_exit_frame_with_exception(self.last_exc_box)
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1058,6 +1058,7 @@
         f.setup_resume_at_op(pc)
         resumereader.consume_boxes(f.get_current_position_info(),
                                    f.registers_i, f.registers_r, f.registers_f)
+        f.handle_rvmprof_enter_on_resume()
     return resumereader.liveboxes, virtualizable_boxes, virtualref_boxes
 
 
@@ -1343,6 +1344,7 @@
         jitcode = jitcodes[jitcode_pos]
         curbh.setposition(jitcode, pc)
         resumereader.consume_one_section(curbh)
+        curbh.handle_rvmprof_enter()
     return curbh
 
 def force_from_resumedata(metainterp_sd, storage, deadframe, vinfo, ginfo):
diff --git a/rpython/rlib/rstacklet.py b/rpython/rlib/rstacklet.py
--- a/rpython/rlib/rstacklet.py
+++ b/rpython/rlib/rstacklet.py
@@ -3,6 +3,7 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import fetch_translated_config
 from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rlib.rvmprof import cintf
 
 DEBUG = False
 
@@ -24,7 +25,12 @@
     def new(self, callback, arg=llmemory.NULL):
         if DEBUG:
             callback = _debug_wrapper(callback)
-        h = self._gcrootfinder.new(self, callback, arg)
+        x = cintf.save_rvmprof_stack()
+        try:
+            cintf.empty_rvmprof_stack()
+            h = self._gcrootfinder.new(self, callback, arg)
+        finally:
+            cintf.restore_rvmprof_stack(x)
         if DEBUG:
             debug.add(h)
         return h
@@ -34,7 +40,11 @@
     def switch(self, stacklet):
         if DEBUG:
             debug.remove(stacklet)
-        h = self._gcrootfinder.switch(stacklet)
+        x = cintf.save_rvmprof_stack()
+        try:
+            h = self._gcrootfinder.switch(stacklet)
+        finally:
+            cintf.restore_rvmprof_stack(x)
         if DEBUG:
             debug.add(h)
         return h
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -6,7 +6,8 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rlib import rthread
+from rpython.rlib import rthread, jit
+from rpython.rlib.objectmodel import we_are_translated
 
 class VMProfPlatformUnsupported(Exception):
     pass
@@ -96,5 +97,60 @@
     return s
 
 def leave_code(s):
+    if not we_are_translated():
+        assert vmprof_tl_stack.getraw() == s
     vmprof_tl_stack.setraw(s.c_next)
     lltype.free(s, flavor='raw')
+
+#
+# JIT notes:
+#
+# - When running JIT-generated assembler code, we have different custom
+#   code to build the VMPROFSTACK, so the functions above are not used.
+#   (It uses kind == VMPROF_JITTED_TAG and the VMPROFSTACK is allocated
+#   in the C stack.)
+#
+# - The jitcode for decorated_jitted_function() in rvmprof.py is
+#   special-cased by jtransform.py to produce this:
+#
+#        rvmprof_code(0, unique_id)
+#        res = inline_call FUNC         <- for func(*args)
+#        rvmprof_code(1, unique_id)
+#        return res
+#
+#   There is no 'catch_exception', but the second 'rvmprof_code' is
+#   meant to be executed even in case there was an exception.  This is
+#   done by a special case in pyjitpl.py and blackhole.py.  The point
+#   is that the above simple pattern can be detected by the blackhole
+#   interp, when it first rebuilds all the intermediate RPython
+#   frames; at that point it needs to call jit_rvmprof_code(0) on all
+#   intermediate RPython frames, so it does pattern matching to
+#   recognize when it must call that and with which 'unique_id' value.
+#
+# - The jitcode opcode 'rvmprof_code' doesn't produce any resop.  When
+#   meta-interpreting, it causes pyjitpl to call jit_rvmprof_code().
+#   As mentioned above, there is logic to call jit_rvmprof_code(1)
+#   even if we exit with an exception, even though there is no
+#   'catch_exception'.  There is similar logic inside the blackhole
+#   interpreter.
+
+
+def jit_rvmprof_code(leaving, unique_id):
+    if leaving == 0:
+        enter_code(unique_id)    # ignore the return value
+    else:
+        s = vmprof_tl_stack.getraw()
+        assert s.c_value == unique_id and s.c_kind == VMPROF_CODE_TAG
+        leave_code(s)
+
+#
+# stacklet support
+
+def save_rvmprof_stack():
+    return vmprof_tl_stack.get_or_make_raw()
+
+def empty_rvmprof_stack():
+    vmprof_tl_stack.setraw(lltype.nullptr(VMPROFSTACK))
+
+def restore_rvmprof_stack(x):
+    vmprof_tl_stack.setraw(x)
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -4,7 +4,7 @@
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
-from rpython.rtyper.lltypesystem import rffi, llmemory
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rweaklist import RWeakListMixin
 
@@ -25,10 +25,16 @@
     def __str__(self):
         return self.msg
 
+class FakeWeakCodeObjectList(object):
+    def add_handle(self, handle):
+        pass
+
 class VMProf(object):
 
     _immutable_fields_ = ['is_enabled?']
 
+    use_weaklist = True # False for tests
+
     def __init__(self):
         "NOT_RPYTHON: use _get_vmprof()"
         self._code_classes = set()
@@ -40,6 +46,7 @@
     def _cleanup_(self):
         self.is_enabled = False
 
+    @jit.dont_look_inside
     @specialize.argtype(1)
     def register_code(self, code, full_name_func):
         """Register the code object.  Call when a new code object is made.
@@ -56,7 +63,7 @@
             self._code_unique_id = uid
             if self.is_enabled:
                 self._write_code_registration(uid, full_name_func(code))
-            else:
+            elif self.use_weaklist:
                 code._vmprof_weak_list.add_handle(code)
 
     def register_code_object_class(self, CodeClass, full_name_func):
@@ -81,12 +88,17 @@
         if CodeClass in self._code_classes:
             return
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
+        immut = CodeClass.__dict__.get('_immutable_fields_', [])
+        CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
         self._code_classes.add(CodeClass)
         #
         class WeakCodeObjectList(RWeakListMixin):
             def __init__(self):
                 self.initialize()
-        CodeClass._vmprof_weak_list = WeakCodeObjectList()
+        if self.use_weaklist:
+            CodeClass._vmprof_weak_list = WeakCodeObjectList()
+        else:
+            CodeClass._vmprof_weak_list = FakeWeakCodeObjectList()
         #
         def gather_all_code_objs():
             all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles()
@@ -102,6 +114,7 @@
         prev = self._gather_all_code_objs
         self._gather_all_code_objs = gather_all_code_objs
 
+    @jit.dont_look_inside
     def enable(self, fileno, interval):
         """Enable vmprof.  Writes go to the given 'fileno'.
         The sampling interval is given by 'interval' as a number of
@@ -122,6 +135,7 @@
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
 
+    @jit.dont_look_inside
     def disable(self):
         """Disable vmprof.
         Raises VMProfError if something goes wrong.
@@ -140,7 +154,8 @@
         if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0:
             raise VMProfError("vmprof buffers full!  disk full or too slow")
 
-def vmprof_execute_code(name, get_code_fn, result_class=None):
+def vmprof_execute_code(name, get_code_fn, result_class=None,
+                        _hack_update_stack_untranslated=False):
     """Decorator to be used on the function that interprets a code object.
 
     'name' must be a unique name.
@@ -150,24 +165,40 @@
 
     'result_class' is ignored (backward compatibility).
     """
+    if _hack_update_stack_untranslated:
+        from rpython.rtyper.annlowlevel import llhelper
+        enter_code = llhelper(lltype.Ptr(
+            lltype.FuncType([lltype.Signed], cintf.PVMPROFSTACK)),
+            cintf.enter_code)
+        leave_code = llhelper(lltype.Ptr(
+            lltype.FuncType([cintf.PVMPROFSTACK], lltype.Void)),
+            cintf.leave_code)
+    else:
+        enter_code = cintf.enter_code
+        leave_code = cintf.leave_code
+
     def decorate(func):
         try:
             _get_vmprof()
         except cintf.VMProfPlatformUnsupported:
             return func
 
+        @jit.oopspec("rvmprof.jitted(unique_id)")
+        def decorated_jitted_function(unique_id, *args):
+            return func(*args)
+
         def decorated_function(*args):
-            # If we are being JITted, we want to skip the trampoline, else the
-            # JIT cannot see through it.
+            unique_id = get_code_fn(*args)._vmprof_unique_id
+            unique_id = rffi.cast(lltype.Signed, unique_id) 
+            # ^^^ removes the "known non-negative" hint for annotation
             if not jit.we_are_jitted():
-                unique_id = get_code_fn(*args)._vmprof_unique_id
-                x = cintf.enter_code(unique_id)
+                x = enter_code(unique_id)
                 try:
                     return func(*args)
                 finally:
-                    cintf.leave_code(x)
+                    leave_code(x)
             else:
-                return func(*args)
+                return decorated_jitted_function(unique_id, *args)
 
         decorated_function.__name__ = func.__name__ + '_rvmprof'
         return decorated_function

From pypy.commits at gmail.com  Tue Aug  9 03:53:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 00:53:44 -0700 (PDT)
Subject: [pypy-commit] pypy inline-blocks: Preserve indentation. Allows us
 to read the diff more easily
Message-ID: <57a98c08.c62f1c0a.88ea8.9705@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: inline-blocks
Changeset: r86095:294e881d0a90
Date: 2016-08-09 09:53 +0200
http://bitbucket.org/pypy/pypy/changeset/294e881d0a90/

Log:	Preserve indentation. Allows us to read the diff more easily

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -193,75 +193,76 @@
                 yield line
 
     def gen_block(self, block):
-        myblocknum = self.blocknum[block]
-        if block in self.inlinable_blocks:
-            # debug comment
-            yield '/* block%d: (inlined) */' % myblocknum
-        else:
-            yield 'block%d:' % myblocknum
-        if block in self.innerloops:
-            for line in self.gen_while_loop_hack(block):
-                yield line
-            return
-        for i, op in enumerate(block.operations):
-            for line in self.gen_op(op):
-                yield line
-        if len(block.exits) == 0:
-            assert len(block.inputargs) == 1
-            # regular return block
-            retval = self.expr(block.inputargs[0])
-            if self.exception_policy != "exc_helper":
-                yield 'RPY_DEBUG_RETURN();'
-            yield 'return %s;' % retval
-            return
-        elif block.exitswitch is None:
-            # single-exit block
-            assert len(block.exits) == 1
-            for op in self.gen_link(block.exits[0]):
-                yield op
-        else:
-            assert not block.canraise
-            # block ending in a switch on a value
-            TYPE = self.lltypemap(block.exitswitch)
-            if TYPE == Bool:
-                expr = self.expr(block.exitswitch)
-                for link in block.exits[:0:-1]:
+        if 1:      # (preserve indentation)
+            myblocknum = self.blocknum[block]
+            if block in self.inlinable_blocks:
+                # debug comment
+                yield '/* block%d: (inlined) */' % myblocknum
+            else:
+                yield 'block%d:' % myblocknum
+            if block in self.innerloops:
+                for line in self.gen_while_loop_hack(block):
+                    yield line
+                return
+            for i, op in enumerate(block.operations):
+                for line in self.gen_op(op):
+                    yield line
+            if len(block.exits) == 0:
+                assert len(block.inputargs) == 1
+                # regular return block
+                retval = self.expr(block.inputargs[0])
+                if self.exception_policy != "exc_helper":
+                    yield 'RPY_DEBUG_RETURN();'
+                yield 'return %s;' % retval
+                return
+            elif block.exitswitch is None:
+                # single-exit block
+                assert len(block.exits) == 1
+                for op in self.gen_link(block.exits[0]):
+                    yield op
+            else:
+                assert not block.canraise
+                # block ending in a switch on a value
+                TYPE = self.lltypemap(block.exitswitch)
+                if TYPE == Bool:
+                    expr = self.expr(block.exitswitch)
+                    for link in block.exits[:0:-1]:
+                        assert link.exitcase in (False, True)
+                        if not link.exitcase:
+                            expr = '!' + expr
+                        yield 'if (%s) {' % expr
+                        for op in self.gen_link(link):
+                            yield '\t' + op
+                        yield '}'
+                    link = block.exits[0]
                     assert link.exitcase in (False, True)
-                    if not link.exitcase:
-                        expr = '!' + expr
-                    yield 'if (%s) {' % expr
                     for op in self.gen_link(link):
-                        yield '\t' + op
+                        yield op
+                elif TYPE in (Signed, Unsigned, SignedLongLong,
+                              UnsignedLongLong, Char, UniChar):
+                    defaultlink = None
+                    expr = self.expr(block.exitswitch)
+                    yield 'switch (%s) {' % self.expr(block.exitswitch)
+                    for link in block.exits:
+                        if link.exitcase == 'default':
+                            defaultlink = link
+                            continue
+                        yield 'case %s:' % self.db.get(link.llexitcase)
+                        for op in self.gen_link(link):
+                            yield '\t' + op
+                        # 'break;' not needed, as gen_link ends in a 'goto'
+                    # Emit default case
+                    yield 'default:'
+                    if defaultlink is None:
+                        yield '\tassert(!"bad switch!!"); abort();'
+                    else:
+                        for op in self.gen_link(defaultlink):
+                            yield '\t' + op
+
                     yield '}'
-                link = block.exits[0]
-                assert link.exitcase in (False, True)
-                for op in self.gen_link(link):
-                    yield op
-            elif TYPE in (Signed, Unsigned, SignedLongLong,
-                            UnsignedLongLong, Char, UniChar):
-                defaultlink = None
-                expr = self.expr(block.exitswitch)
-                yield 'switch (%s) {' % self.expr(block.exitswitch)
-                for link in block.exits:
-                    if link.exitcase == 'default':
-                        defaultlink = link
-                        continue
-                    yield 'case %s:' % self.db.get(link.llexitcase)
-                    for op in self.gen_link(link):
-                        yield '\t' + op
-                    # 'break;' not needed, as gen_link ends in a 'goto'
-                # Emit default case
-                yield 'default:'
-                if defaultlink is None:
-                    yield '\tassert(!"bad switch!!"); abort();'
                 else:
-                    for op in self.gen_link(defaultlink):
-                        yield '\t' + op
-
-                yield '}'
-            else:
-                raise TypeError("exitswitch type not supported"
-                                "  Got %r" % (TYPE,))
+                    raise TypeError("exitswitch type not supported"
+                                    "  Got %r" % (TYPE,))
 
     def gen_link(self, link):
         "Generate the code to jump across the given Link."

From pypy.commits at gmail.com  Tue Aug  9 04:23:19 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 09 Aug 2016 01:23:19 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: another test testing unpack
 operation (mostly integer)
Message-ID: <57a992f7.28eac20a.8f95a.8def@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86096:5d676fb9307f
Date: 2016-08-09 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/5d676fb9307f/

Log:	another test testing unpack operation (mostly integer)

diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -754,17 +754,36 @@
             vars['x'] = v
         packs = '\n        '.join(pack)
         resvar = suffix + '{'+suffix+'}'
+
+        # format the resoperations, take care that the lhs of =
+        # is formated later with a new variable name
+        unpackops = unpack
+        if isinstance(unpack, str):
+            unpackops = [unpack]
+        unpacksf = []
+        for up in unpackops:
+            lhs, rhs = up.split("=")
+            rhsf = rhs.format(**vars)
+            newvar('i'); newvar('f'); newvar('v')
+            lhsf = lhs.format(**vars)
+            unpacksf.append(lhsf + '=' + rhsf)
+        unpacks = '\n        '.join(unpacksf)
+
         source = '''
         [{args}]
         label({args}, descr=targettoken)
         {packs}
-        {unpack}
+        {unpacks}
         finish({resvar}, descr=finaldescr)
-        '''.format(args=','.join(args),packs=packs, unpack=unpack.format(**vars),
+        '''.format(args=','.join(args),packs=packs, unpacks=unpacks,
                    resvar=resvar.format(**vars))
-        loop = parse(source, namespace={'targettoken': targettoken,
-                                        'finaldescr': finaldescr})
+        print(source)
+        return self._compile_and_run(source, args_values, float,
+                ns={'targettoken': targettoken, 'finaldescr': finaldescr})
 
+
+    def _compile_and_run(self, source, args_values, float=True, ns={}):
+        loop = parse(source, namespace=ns)
         cpu = self.CPUClass(rtyper=None, stats=None)
         cpu.setup_once()
         #
@@ -792,11 +811,37 @@
         ## integer unpack (byte)
         for i in range(16):
             op = "i{i} = vec_unpack_i({x}, %d, 1)" % i
-            assert self.run_unpack(op, "[16xi8]", {'x': [127,1]*8}, float=False) == (127 if i%2==0 else 1)
+            assert self.run_unpack(op, "[16xi8]", {'x': [127,1]*8}, float=False) == \
+                   (127 if i%2==0 else 1)
             if i < 8:
-                assert self.run_unpack(op, "[2xi16]", {'x': [2**15-1,0]*4}, float=False) == (2**15-1 if i%2==0 else 0)
+                assert self.run_unpack(op, "[8xi16]", {'x': [2**15-1,0]*4}, float=False) == \
+                       (2**15-1 if i%2==0 else 0)
             if i < 4:
-                assert self.run_unpack(op, "[2xi32]", {'x': [2**31-1,0]*4}, float=False) == (2**31-1 if i%2==0 else 0)
+                assert self.run_unpack(op, "[4xi32]", {'x': [2**31-1,0]*4}, float=False) == \
+                       (2**31-1 if i%2==0 else 0)
+
+    def test_unpack_several(self):
+        # count == 2
+        values = [1,2,3,4]
+        for i,v in enumerate(values):
+            j = (i // 2) * 2
+            op = ["v{v}[2xi32] = vec_unpack_i({x}, %d, 2)" % j,
+                  "i{i} = vec_unpack_i(v{v}[2xi32], %d, 1)" % i]
+            assert self.run_unpack(op, "[4xi32]", {'x': values}, float=False) == v
+
+        values = [1,2,3,4,5,6,7,8]
+        for i,v in enumerate(values):
+            j = (i // 4) * 4
+            op = ["v{v}[4xi16] = vec_unpack_i({x}, %d, 4)" % j,
+                  "i{i} = vec_unpack_i(v{v}[4xi16], %d, 1)" % i]
+            assert self.run_unpack(op, "[8xi16]", {'x': values}, float=False) == v
+
+        values = [1,2,3,4,5,6,7,8] * 2
+        for i,v in enumerate(values):
+            j = (i // 8) * 8
+            op = ["v{v}[8xi8] = vec_unpack_i({x}, %d, 8)" % j,
+                  "i{i} = vec_unpack_i(v{v}[8xi8], %d, 1)" % i]
+            assert self.run_unpack(op, "[16xi8]", {'x': values}, float=False) == v
 
 
 class TestLLtype(LLJitMixin, VectorizeTests):

From pypy.commits at gmail.com  Tue Aug  9 04:32:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 01:32:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: Factor out some of the timeout logic.
 Should be a no-op.
Message-ID: <57a9950d.a710c20a.3fa53.97fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86097:c6334dab174d
Date: 2016-08-09 10:31 +0200
http://bitbucket.org/pypy/pypy/changeset/c6334dab174d/

Log:	Factor out some of the timeout logic. Should be a no-op.

diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -846,22 +846,27 @@
         if res < 0:
             raise self.error_handler()
 
+    def wait_for_data(self, for_writing):
+        timeout = self._select(for_writing)
+        if timeout != 0:
+            if timeout == 1:
+                raise SocketTimeout
+            else:
+                raise self.error_handler()
+
     def recv(self, buffersize, flags=0):
         """Receive up to buffersize bytes from the socket.  For the optional
         flags argument, see the Unix manual.  When no data is available, block
         until at least one byte is available or until the remote end is closed.
         When the remote end is closed and all data is read, return the empty
         string."""
-        timeout = self._select(False)
-        if timeout == 1:
-            raise SocketTimeout
-        elif timeout == 0:
-            with rffi.scoped_alloc_buffer(buffersize) as buf:
-                read_bytes = _c.socketrecv(self.fd,
-                                           rffi.cast(rffi.VOIDP, buf.raw),
-                                           buffersize, flags)
-                if read_bytes >= 0:
-                    return buf.str(read_bytes)
+        self.wait_for_data(False)
+        with rffi.scoped_alloc_buffer(buffersize) as buf:
+            read_bytes = _c.socketrecv(self.fd,
+                                       rffi.cast(rffi.VOIDP, buf.raw),
+                                       buffersize, flags)
+            if read_bytes >= 0:
+                return buf.str(read_bytes)
         raise self.error_handler()
 
     def recvinto(self, rwbuffer, nbytes, flags=0):
@@ -874,26 +879,23 @@
         """Like recv(buffersize, flags) but also return the sender's
         address."""
         read_bytes = -1
-        timeout = self._select(False)
-        if timeout == 1:
-            raise SocketTimeout
-        elif timeout == 0:
-            with rffi.scoped_alloc_buffer(buffersize) as buf:
-                address, addr_p, addrlen_p = self._addrbuf()
-                try:
-                    read_bytes = _c.recvfrom(self.fd, buf.raw, buffersize, flags,
-                                             addr_p, addrlen_p)
-                    addrlen = rffi.cast(lltype.Signed, addrlen_p[0])
-                finally:
-                    lltype.free(addrlen_p, flavor='raw')
-                    address.unlock()
-                if read_bytes >= 0:
-                    if addrlen:
-                        address.addrlen = addrlen
-                    else:
-                        address = None
-                    data = buf.str(read_bytes)
-                    return (data, address)
+        self.wait_for_data(False)
+        with rffi.scoped_alloc_buffer(buffersize) as buf:
+            address, addr_p, addrlen_p = self._addrbuf()
+            try:
+                read_bytes = _c.recvfrom(self.fd, buf.raw, buffersize, flags,
+                                         addr_p, addrlen_p)
+                addrlen = rffi.cast(lltype.Signed, addrlen_p[0])
+            finally:
+                lltype.free(addrlen_p, flavor='raw')
+                address.unlock()
+            if read_bytes >= 0:
+                if addrlen:
+                    address.addrlen = addrlen
+                else:
+                    address = None
+                data = buf.str(read_bytes)
+                return (data, address)
         raise self.error_handler()
 
     def recvfrom_into(self, rwbuffer, nbytes, flags=0):
@@ -903,12 +905,8 @@
 
     def send_raw(self, dataptr, length, flags=0):
         """Send data from a CCHARP buffer."""
-        res = -1
-        timeout = self._select(True)
-        if timeout == 1:
-            raise SocketTimeout
-        elif timeout == 0:
-            res = _c.send(self.fd, dataptr, length, flags)
+        self.wait_for_data(True)
+        res = _c.send(self.fd, dataptr, length, flags)
         if res < 0:
             raise self.error_handler()
         return res
@@ -942,15 +940,11 @@
     def sendto(self, data, flags, address):
         """Like send(data, flags) but allows specifying the destination
         address.  (Note that 'flags' is mandatory here.)"""
-        res = -1
-        timeout = self._select(True)
-        if timeout == 1:
-            raise SocketTimeout
-        elif timeout == 0:
-            addr = address.lock()
-            res = _c.sendto(self.fd, data, len(data), flags,
-                            addr, address.addrlen)
-            address.unlock()
+        self.wait_for_data(True)
+        addr = address.lock()
+        res = _c.sendto(self.fd, data, len(data), flags,
+                        addr, address.addrlen)
+        address.unlock()
         if res < 0:
             raise self.error_handler()
         return res

From pypy.commits at gmail.com  Tue Aug  9 04:52:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 01:52:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix socket.recvfrom() so that it takes
 advantage of the fact that, 
Message-ID: <57a999c3.a111c20a.a2129.8dba@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86098:e53ea5c9c384
Date: 2016-08-09 10:51 +0200
http://bitbucket.org/pypy/pypy/changeset/e53ea5c9c384/

Log:	Fix socket.recvfrom() so that it takes advantage of the fact that,
	nowadays, a lot of buffers have a get_raw_address()

diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -862,23 +862,30 @@
         string."""
         self.wait_for_data(False)
         with rffi.scoped_alloc_buffer(buffersize) as buf:
-            read_bytes = _c.socketrecv(self.fd,
-                                       rffi.cast(rffi.VOIDP, buf.raw),
-                                       buffersize, flags)
+            read_bytes = _c.socketrecv(self.fd, buf.raw, buffersize, flags)
             if read_bytes >= 0:
                 return buf.str(read_bytes)
         raise self.error_handler()
 
     def recvinto(self, rwbuffer, nbytes, flags=0):
-        buf = self.recv(nbytes, flags)
-        rwbuffer.setslice(0, buf)
-        return len(buf)
+        try:
+            rwbuffer.get_raw_address()
+        except ValueError:
+            buf = self.recv(nbytes, flags)
+            rwbuffer.setslice(0, buf)
+            return len(buf)
+        else:
+            self.wait_for_data(False)
+            raw = rwbuffer.get_raw_address()
+            read_bytes = _c.socketrecv(self.fd, raw, nbytes, flags)
+            if read_bytes >= 0:
+                return read_bytes
+            raise self.error_handler()
 
     @jit.dont_look_inside
     def recvfrom(self, buffersize, flags=0):
         """Like recv(buffersize, flags) but also return the sender's
         address."""
-        read_bytes = -1
         self.wait_for_data(False)
         with rffi.scoped_alloc_buffer(buffersize) as buf:
             address, addr_p, addrlen_p = self._addrbuf()
@@ -899,9 +906,30 @@
         raise self.error_handler()
 
     def recvfrom_into(self, rwbuffer, nbytes, flags=0):
-        buf, addr = self.recvfrom(nbytes, flags)
-        rwbuffer.setslice(0, buf)
-        return len(buf), addr
+        try:
+            rwbuffer.get_raw_address()
+        except ValueError:
+            buf, addr = self.recvfrom(nbytes, flags)
+            rwbuffer.setslice(0, buf)
+            return len(buf), addr
+        else:
+            self.wait_for_data(False)
+            address, addr_p, addrlen_p = self._addrbuf()
+            try:
+                raw = rwbuffer.get_raw_address()
+                read_bytes = _c.recvfrom(self.fd, raw, nbytes, flags,
+                                         addr_p, addrlen_p)
+                addrlen = rffi.cast(lltype.Signed, addrlen_p[0])
+            finally:
+                lltype.free(addrlen_p, flavor='raw')
+                address.unlock()
+            if read_bytes >= 0:
+                if addrlen:
+                    address.addrlen = addrlen
+                else:
+                    address = None
+                return (read_bytes, address)
+            raise self.error_handler()
 
     def send_raw(self, dataptr, length, flags=0):
         """Send data from a CCHARP buffer."""
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -119,25 +119,111 @@
     s1.close()
     s2.close()
 
-def test_socketpair_recvinto():
+def test_socketpair_recvinto_1():
     class Buffer:
         def setslice(self, start, string):
             self.x = string
 
-        def as_str(self):
-            return self.x
+        def get_raw_address(self):
+            raise ValueError
 
     if sys.platform == "win32":
         py.test.skip('No socketpair on Windows')
     s1, s2 = socketpair()
     buf = Buffer()
     s1.sendall('?')
-    s2.recvinto(buf, 1)
-    assert buf.as_str() == '?'
+    n = s2.recvinto(buf, 1)
+    assert n == 1
+    assert buf.x == '?'
     count = s2.send('x'*99)
     assert 1 <= count <= 99
-    s1.recvinto(buf, 100)
-    assert buf.as_str() == 'x'*count
+    n = s1.recvinto(buf, 100)
+    assert n == count
+    assert buf.x == 'x'*count
+    s1.close()
+    s2.close()
+
+def test_socketpair_recvinto_2():
+    class Buffer:
+        def __init__(self):
+            self._p = lltype.malloc(rffi.CCHARP.TO, 100, flavor='raw',
+                                    track_allocation=False)
+
+        def _as_str(self, count):
+            return rffi.charpsize2str(self._p, count)
+
+        def get_raw_address(self):
+            return self._p
+
+    if sys.platform == "win32":
+        py.test.skip('No socketpair on Windows')
+    s1, s2 = socketpair()
+    buf = Buffer()
+    s1.sendall('?')
+    n = s2.recvinto(buf, 1)
+    assert n == 1
+    assert buf._as_str(1) == '?'
+    count = s2.send('x'*99)
+    assert 1 <= count <= 99
+    n = s1.recvinto(buf, 100)
+    assert n == count
+    assert buf._as_str(n) == 'x'*count
+    s1.close()
+    s2.close()
+
+def test_socketpair_recvfrom_into_1():
+    class Buffer:
+        def setslice(self, start, string):
+            self.x = string
+
+        def get_raw_address(self):
+            raise ValueError
+
+    if sys.platform == "win32":
+        py.test.skip('No socketpair on Windows')
+    s1, s2 = socketpair()
+    buf = Buffer()
+    s1.sendall('?')
+    n, addr = s2.recvfrom_into(buf, 1)
+    assert n == 1
+    assert addr is None
+    assert buf.x == '?'
+    count = s2.send('x'*99)
+    assert 1 <= count <= 99
+    n, addr = s1.recvfrom_into(buf, 100)
+    assert n == count
+    assert addr is None
+    assert buf.x == 'x'*count
+    s1.close()
+    s2.close()
+
+def test_socketpair_recvfrom_into_2():
+    class Buffer:
+        def __init__(self):
+            self._p = lltype.malloc(rffi.CCHARP.TO, 100, flavor='raw',
+                                    track_allocation=False)
+
+        def _as_str(self, count):
+            return rffi.charpsize2str(self._p, count)
+
+        def get_raw_address(self):
+            return self._p
+
+    if sys.platform == "win32":
+        py.test.skip('No socketpair on Windows')
+    s1, s2 = socketpair()
+    buf = Buffer()
+    s1.sendall('?')
+    n, addr = s2.recvfrom_into(buf, 1)
+    assert n == 1
+    assert addr is None
+    assert buf._as_str(1) == '?'
+    count = s2.send('x'*99)
+    assert 1 <= count <= 99
+    n, addr = s1.recvfrom_into(buf, 100)
+    assert n == count
+    assert addr is None
+    assert buf._as_str(n) == 'x'*count
     s1.close()
     s2.close()
 

From pypy.commits at gmail.com  Tue Aug  9 05:15:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 02:15:44 -0700 (PDT)
Subject: [pypy-commit] pypy inline-blocks: Close branch inline-blocks
Message-ID: <57a99f40.a111c20a.a2129.97ec@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: inline-blocks
Changeset: r86099:c5053a699285
Date: 2016-08-09 11:14 +0200
http://bitbucket.org/pypy/pypy/changeset/c5053a699285/

Log:	Close branch inline-blocks


From pypy.commits at gmail.com  Tue Aug  9 05:16:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 02:16:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in inline-blocks (pull request
 #467)
Message-ID: <57a99f67.8bc71c0a.7f7bb.b721@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r86100:4c64ef74c612
Date: 2016-08-09 11:14 +0200
http://bitbucket.org/pypy/pypy/changeset/4c64ef74c612/

Log:	Merged in inline-blocks (pull request #467)

	Inline gotos to blocks with only one predecessor.

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -2,7 +2,7 @@
 from rpython.translator.c.support import cdecl
 from rpython.translator.c.support import llvalue_from_constant, gen_assignments
 from rpython.translator.c.support import c_string_constant, barebonearray
-from rpython.flowspace.model import Variable, Constant
+from rpython.flowspace.model import Variable, Constant, mkentrymap
 from rpython.rtyper.lltypesystem.lltype import (Ptr, Void, Bool, Signed, Unsigned,
     SignedLongLong, Float, UnsignedLongLong, Char, UniChar, ContainerType,
     Array, FixedSizeArray, ForwardReference, FuncType)
@@ -173,17 +173,37 @@
 
     def cfunction_body(self):
         graph = self.graph
-        yield 'goto block0;'    # to avoid a warning "this label is not used"
+        # yield 'goto block0;'    # to avoid a warning "this label is not used"
 
-        # generate the body of each block
+        # Locate blocks with a single predecessor, which can be written
+        # inline in place of a "goto":
+        entrymap = mkentrymap(graph)
+        self.inlinable_blocks = {
+            block for block in entrymap if len(entrymap[block]) == 1}
+
+        yield ''
+        for line in self.gen_goto(graph.startblock):
+            yield line
+
+        # Only blocks left are those that have more than one predecessor.
         for block in graph.iterblocks():
+            if block in self.inlinable_blocks:
+                continue
+            for line in self.gen_block(block):
+                yield line
+
+    def gen_block(self, block):
+        if 1:      # (preserve indentation)
             myblocknum = self.blocknum[block]
-            yield ''
-            yield 'block%d:' % myblocknum
+            if block in self.inlinable_blocks:
+                # debug comment
+                yield '/* block%d: (inlined) */' % myblocknum
+            else:
+                yield 'block%d:' % myblocknum
             if block in self.innerloops:
                 for line in self.gen_while_loop_hack(block):
                     yield line
-                continue
+                return
             for i, op in enumerate(block.operations):
                 for line in self.gen_op(op):
                     yield line
@@ -194,7 +214,7 @@
                 if self.exception_policy != "exc_helper":
                     yield 'RPY_DEBUG_RETURN();'
                 yield 'return %s;' % retval
-                continue
+                return
             elif block.exitswitch is None:
                 # single-exit block
                 assert len(block.exits) == 1
@@ -256,12 +276,25 @@
             assignments.append((a2typename, dest, src))
         for line in gen_assignments(assignments):
             yield line
-        label = 'block%d' % self.blocknum[link.target]
-        if link.target in self.innerloops:
-            loop = self.innerloops[link.target]
+        for line in self.gen_goto(link.target, link):
+            yield line
+
+    def gen_goto(self, target, link=None):
+        """Recursively expand block with inlining or goto.
+
+        Blocks that have only one predecessor are inlined directly, all others
+        are reached via goto.
+        """
+        label = 'block%d' % self.blocknum[target]
+        if target in self.innerloops:
+            loop = self.innerloops[target]
             if link is loop.links[-1]:   # link that ends a loop
                 label += '_back'
-        yield 'goto %s;' % label
+        if target in self.inlinable_blocks:
+            for line in self.gen_block(target):
+                yield line
+        else:
+            yield 'goto %s;' % label
 
     def gen_op(self, op):
         macro = 'OP_%s' % op.opname.upper()
diff --git a/rpython/translator/c/test/test_genc.py b/rpython/translator/c/test/test_genc.py
--- a/rpython/translator/c/test/test_genc.py
+++ b/rpython/translator/c/test/test_genc.py
@@ -1,4 +1,5 @@
 import ctypes
+import re
 from collections import OrderedDict
 
 import py
@@ -13,6 +14,7 @@
 from rpython.rtyper.lltypesystem.rstr import STR
 from rpython.tool.nullpath import NullPyPathLocal
 from rpython.translator.c import genc
+from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator.interactive import Translation
 from rpython.translator.translator import TranslationContext, graphof
 
@@ -604,3 +606,23 @@
     else:
         assert 0, "the call was not found in the C source"
     assert 'PYPY_INHIBIT_TAIL_CALL();' in lines[i+1]
+
+def get_generated_c_source(fn, types):
+    """Return the generated C source for fn."""
+    t = Translation(fn, types, backend="c")
+    t.annotate()
+    merge_if_blocks(t.driver.translator.graphs[0])
+    c_filename_path = t.source_c()
+    return t.driver.cbuilder.c_source_filename.join('..',
+                              'rpython_translator_c_test.c').read()
+
+def test_generated_c_source_no_gotos():
+    # We want simple functions to have no indirection/goto.
+    # Instead, PyPy can inline blocks when they aren't reused.
+
+    def main(x):
+        return x + 1
+
+    c_src = get_generated_c_source(main, [int])
+    assert 'goto' not in c_src
+    assert not re.search(r'block\w*:(?! \(inlined\))', c_src)

From pypy.commits at gmail.com  Tue Aug  9 05:36:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 02:36:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: Some missing keepalives (I think)
Message-ID: <57a9a40b.eeb8c20a.6ebe3.a8cb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86101:f5b4cb2bb733
Date: 2016-08-09 11:35 +0200
http://bitbucket.org/pypy/pypy/changeset/f5b4cb2bb733/

Log:	Some missing keepalives (I think)

diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -878,6 +878,7 @@
             self.wait_for_data(False)
             raw = rwbuffer.get_raw_address()
             read_bytes = _c.socketrecv(self.fd, raw, nbytes, flags)
+            keepalive_until_here(rwbuffer)
             if read_bytes >= 0:
                 return read_bytes
             raise self.error_handler()
@@ -919,6 +920,7 @@
                 raw = rwbuffer.get_raw_address()
                 read_bytes = _c.recvfrom(self.fd, raw, nbytes, flags,
                                          addr_p, addrlen_p)
+                keepalive_until_here(rwbuffer)
                 addrlen = rffi.cast(lltype.Signed, addrlen_p[0])
             finally:
                 lltype.free(addrlen_p, flavor='raw')

From pypy.commits at gmail.com  Tue Aug  9 05:46:14 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 09 Aug 2016 02:46:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove outdated comment
Message-ID: <57a9a666.81cb1c0a.d1959.c2f6@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86102:6a3c7c1c12ef
Date: 2016-08-09 11:45 +0200
http://bitbucket.org/pypy/pypy/changeset/6a3c7c1c12ef/

Log:	remove outdated comment

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -173,7 +173,6 @@
 
     def cfunction_body(self):
         graph = self.graph
-        # yield 'goto block0;'    # to avoid a warning "this label is not used"
 
         # Locate blocks with a single predecessor, which can be written
         # inline in place of a "goto":

From pypy.commits at gmail.com  Tue Aug  9 09:59:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 09 Aug 2016 06:59:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async-translate: merged py3.5 changes
Message-ID: <57a9e1a6.4bc41c0a.52bee.2efd@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async-translate
Changeset: r86103:3db60dde36a5
Date: 2016-08-09 14:13 +0200
http://bitbucket.org/pypy/pypy/changeset/3db60dde36a5/

Log:	merged py3.5 changes

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -18,7 +18,6 @@
 from pypy.interpreter.nestedscope import Cell
 from pypy.interpreter.pycode import PyCode, BytecodeCorruption
 from pypy.tool.stdlib_opcode import bytecode_spec
-from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 
 CANNOT_CATCH_MSG = ("catching classes that don't inherit from BaseException "
                     "is not allowed in 3.x")
@@ -1391,9 +1390,8 @@
         self.pushvalue(w_sum)
 
     def BUILD_TUPLE_UNPACK(self, itemcount, next_instr):
-        space = self.space
         w_sum_list = list_unpack_helper(self, itemcount)
-        self.pushvalue(space.newtuple(w_sum_list))
+        self.pushvalue(self.space.newtuple(w_sum_list))
 
     def BUILD_LIST_UNPACK(self, itemcount, next_instr):
         w_sum = list_unpack_helper(self, itemcount)
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -256,7 +256,63 @@
             return a, b, c, d
         """
         assert self.codetest(code, "f", [1, 2], {"d" : 4, "c" : 3}) == (1, 2, 3, 4)
-
+    
+    def test_build_set_unpack(self):
+        code = """ def f():
+            return {*range(4), 4, *(5, 6, 7)}
+        """
+        space = self.space
+        res = self.codetest(code, "f", [])
+        l_res = space.call_function(space.w_list, res)
+        assert space.unwrap(l_res) == [0, 1, 2, 3, 4, 5, 6, 7]
+        
+    def test_build_tuple_unpack(self):
+        code = """ def f():
+            return (*range(4), 4)
+        """
+        assert self.codetest(code, "f", []) == (0, 1, 2, 3, 4)
+    
+    def test_build_list_unpack(self):
+        code = """ def f():
+            return [*range(4), 4]
+        """
+        assert self.codetest(code, "f", []) == [0, 1, 2, 3, 4]
+    
+    def test_build_map_unpack(self):
+        code = """
+        def f():
+            return {'x': 1, **{'y': 2}}
+        def g():
+            return {**()}
+        """
+        assert self.codetest(code, "f", []) == {'x': 1, 'y': 2}
+        res = self.codetest(code, 'g', [])
+        assert "TypeError:" in res
+        assert "'tuple' object is not a mapping" in res
+    
+    def test_build_map_unpack_with_call(self):
+        code = """
+        def f(a,b,c,d):
+            return a+b,c+d
+        def g1():
+            return f(**{'a': 1, 'c': 3}, **{'b': 2, 'd': 4})
+        def g2():
+            return f(**{'a': 1, 'c': 3}, **[])
+        def g3():
+            return f(**{'a': 1, 'c': 3}, **{1: 3})
+        def g4():
+            return f(**{'a': 1, 'c': 3}, **{'a': 2})
+        """
+        assert self.codetest(code, "g1", []) == (3, 7)
+        resg2 = self.codetest(code, 'g2', [])
+        assert "TypeError:" in resg2
+        assert "'list' object is not a mapping" in resg2
+        resg3 = self.codetest(code, 'g3', [])
+        assert "TypeError:" in resg3
+        assert "keywords must be strings" in resg3
+        resg4 = self.codetest(code, 'g4', [])
+        assert "TypeError:" in resg4
+        assert "f() got multiple values for keyword argument 'a'" in resg4
 
 
 class AppTestInterpreter: 
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -0,0 +1,14 @@
+class AppTestAsyncIO(object):
+    
+    spaceconfig = dict(usemodules=["select","_socket","thread","signal","struct","_multiprocessing","array","_posixsubprocess","fcntl","unicodedata"])
+    
+    def test_gil_issue(self):
+        # the problem occured at await asyncio.open_connection after calling run_until_complete
+        """
+        import encodings.idna
+        import asyncio
+        async def f():
+            reader, writer = await asyncio.open_connection('example.com', 80)
+        
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(f())"""
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -147,7 +147,8 @@
 def set_sentinel(space):
     """Set a sentinel lock that will be released when the current thread
 state is finalized (after it is untied from the interpreter)."""
-    return space.wrap(Lock(space))
+    lock = allocate_lock(space)
+    return lock
 
 class W_RLock(W_Root):
     def __init__(self, space):

From pypy.commits at gmail.com  Tue Aug  9 09:59:05 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 09 Aug 2016 06:59:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async-translate: fixed translatable
 build_map_unpack(_with_call), passes tests
Message-ID: <57a9e1a9.482cc20a.13e8b.1b60@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async-translate
Changeset: r86104:ae4d7b55324b
Date: 2016-08-09 14:37 +0200
http://bitbucket.org/pypy/pypy/changeset/ae4d7b55324b/

Log:	fixed translatable build_map_unpack(_with_call), passes tests

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -45,10 +45,10 @@
 
     return func_with_new_name(opimpl, "opcode_impl_for_%s" % operationname)
 
-def get_func_desc(func):
-    if self.space.type(func) is function.Function:
+def get_func_desc(space, func):
+    if isinstance(func,function.Function):
         return "()"
-    elif self.space.type(func) is function.Method:
+    elif isinstance(func, function.Method):
         return "()"
     else:
         return " object";
@@ -1404,7 +1404,7 @@
         w_dict = space.newdict()
         for i in range(num_maps, 0, -1):
             w_item = self.peekvalue(i-1)
-            if space.lookup(w_item, '__getitem__') is None:
+            if not space.ismapping_w(w_item):
                 raise oefmt(space.w_TypeError,
                         "'%T' object is not a mapping", w_item)
             iterator = w_item.iterkeys()
@@ -1416,13 +1416,13 @@
                     err_fun = self.peekvalue(num_maps + function_location-1)
                     raise oefmt(space.w_TypeError,
                         "%N%s keywords must be strings", err_fun,
-                                                         get_func_desc(err_fun))
+                                                         get_func_desc(space, err_fun))
                 if space.is_true(space.contains(w_dict,w_key)):
                     err_fun = self.peekvalue(num_maps + function_location-1)
                     err_arg = w_key
                     raise oefmt(space.w_TypeError,
-                        "%N%s got multiple values for keyword argument %s",
-                          err_fun, get_func_desc(err_fun), err_arg)
+                        "%N%s got multiple values for keyword argument '%s'",
+                          err_fun, get_func_desc(space, err_fun), space.str_w(err_arg))
             space.call_method(w_dict, 'update', w_item)
         while num_maps != 0:
             self.popvalue()
@@ -1434,7 +1434,7 @@
         w_dict = space.newdict()
         for i in range(itemcount, 0, -1):
             w_item = self.peekvalue(i-1)
-            if space.lookup(w_item, '__getitem__') is None:
+            if not space.ismapping_w(w_item):
                 raise oefmt(self.space.w_TypeError,
                         "'%T' object is not a mapping", w_item)
             space.call_method(w_dict, 'update', w_item)

From pypy.commits at gmail.com  Tue Aug  9 09:59:07 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 09 Aug 2016 06:59:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async-translate: some more translation
 issues resolved
Message-ID: <57a9e1ab.e2efc20a.3d189.3191@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async-translate
Changeset: r86105:2ac7d3003c50
Date: 2016-08-09 15:58 +0200
http://bitbucket.org/pypy/pypy/changeset/2ac7d3003c50/

Log:	some more translation issues resolved

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -59,7 +59,6 @@
     w_sum = space.newlist([], sizehint=itemcount)
     for i in range(itemcount, 0, -1):
         w_item = frame.peekvalue(i-1)
-        #items = frame.space.fixedview(w_item)
         w_sum.extend(w_item)
     while itemcount != 0:
         frame.popvalue()
@@ -1390,8 +1389,9 @@
         self.pushvalue(w_sum)
 
     def BUILD_TUPLE_UNPACK(self, itemcount, next_instr):
-        w_sum_list = list_unpack_helper(self, itemcount)
-        self.pushvalue(self.space.newtuple(w_sum_list))
+        w_list = list_unpack_helper(self, itemcount)
+        items = [w_obj for w_obj in w_list.getitems_unroll()]
+        self.pushvalue(self.space.newtuple(items))
 
     def BUILD_LIST_UNPACK(self, itemcount, next_instr):
         w_sum = list_unpack_helper(self, itemcount)
@@ -1407,7 +1407,7 @@
             if not space.ismapping_w(w_item):
                 raise oefmt(space.w_TypeError,
                         "'%T' object is not a mapping", w_item)
-            iterator = w_item.iterkeys()
+            iterator = w_item.iterkeys(w_item)
             while True:
                 w_key = iterator.next_key()
                 if w_key is None:
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -1,9 +1,13 @@
 class AppTestAsyncIO(object):
     
-    spaceconfig = dict(usemodules=["select","_socket","thread","signal","struct","_multiprocessing","array","_posixsubprocess","fcntl","unicodedata"])
+    spaceconfig = dict(usemodules=["select","_socket","thread","signal",
+                                   "struct","_multiprocessing","array",
+                                   "_posixsubprocess","fcntl",
+                                   "unicodedata"])
     
     def test_gil_issue(self):
-        # the problem occured at await asyncio.open_connection after calling run_until_complete
+        # the problem occured at await asyncio.open_connection
+        # after calling run_until_complete
         """
         import encodings.idna
         import asyncio
@@ -11,4 +15,6 @@
             reader, writer = await asyncio.open_connection('example.com', 80)
         
         loop = asyncio.get_event_loop()
-        loop.run_until_complete(f())"""
+        loop.run_until_complete(f())
+        print("done with async loop")
+        """
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -82,7 +82,7 @@
                    for key in self.cache.keys()]
         return space.newlist(items_w)
 
-    def iterkeys(self, space):
+    def iteratekeys(self, space):
         return space.iter(self.keys(space))
 
     def itervalues(self, space):
@@ -106,11 +106,11 @@
     'zip_dict',
     __getitem__ = interp2app(W_ZipCache.getitem),
     __contains__ = interp2app(W_ZipCache.contains),
-    __iter__ = interp2app(W_ZipCache.iterkeys),
+    __iter__ = interp2app(W_ZipCache.iteratekeys),
     items = interp2app(W_ZipCache.items),
     iteritems = interp2app(W_ZipCache.iteritems),
     keys = interp2app(W_ZipCache.keys),
-    iterkeys = interp2app(W_ZipCache.iterkeys),
+    iterkeys = interp2app(W_ZipCache.iteratekeys),
     values = interp2app(W_ZipCache.values),
     itervalues = interp2app(W_ZipCache.itervalues),
     clear = interp2app(W_ZipCache.clear),

From pypy.commits at gmail.com  Tue Aug  9 10:22:56 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 09 Aug 2016 07:22:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async-translate: iterkeys translation
 issue resolved
Message-ID: <57a9e740.94a51c0a.3a1e7.3315@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async-translate
Changeset: r86106:6f4bb70dde25
Date: 2016-08-09 16:22 +0200
http://bitbucket.org/pypy/pypy/changeset/6f4bb70dde25/

Log:	iterkeys translation issue resolved

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1407,7 +1407,7 @@
             if not space.ismapping_w(w_item):
                 raise oefmt(space.w_TypeError,
                         "'%T' object is not a mapping", w_item)
-            iterator = w_item.iterkeys(w_item)
+            iterator = w_item.iterkeys()
             while True:
                 w_key = iterator.next_key()
                 if w_key is None:

From pypy.commits at gmail.com  Tue Aug  9 10:46:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 07:46:18 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Starting thread support:
 RPY_REVDB_EMIT() should be called mostly when
Message-ID: <57a9ecba.c70a1c0a.38ae3.43ce@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86107:8009adcd405a
Date: 2016-08-09 16:45 +0200
http://bitbucket.org/pypy/pypy/changeset/8009adcd405a/

Log:	Starting thread support: RPY_REVDB_EMIT() should be called mostly
	when we hold the GIL, but ensuring that is messy. For now, we simply
	use our own lock here.

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -11,6 +11,7 @@
 #include <ctype.h>
 #include <signal.h>
 #include <search.h>
+#include <sched.h>
 
 #ifdef __linux__
 #  define HAVE_PERSONALITY
@@ -119,17 +120,27 @@
         setup_record_mode(*argc_p, *argv_p);
 }
 
+static void reverse_db_lock_and_flush(void)
+{
+    _RPY_REVDB_LOCK();
+    rpy_reverse_db_flush();
+    _RPY_REVDB_UNLOCK();
+}
+
 RPY_EXTERN
 void rpy_reverse_db_teardown(void)
 {
     uint64_t stop_points;
-    if (RPY_RDB_REPLAY) {
+    if (!RPY_RDB_REPLAY) {
+        _RPY_REVDB_LOCK();
+    }
+    else {
         /* hack: prevents RPY_REVDB_EMIT() from calling
            rpy_reverse_db_fetch(), which has nothing more to fetch now */
         rpy_revdb.buf_limit += 1;
     }
-    RPY_REVDB_EMIT(stop_points = rpy_revdb.stop_point_seen; ,
-                   uint64_t _e, stop_points);
+    _RPY_REVDB_EMIT_L(stop_points = rpy_revdb.stop_point_seen; ,
+                      uint64_t _e, stop_points, /*must_lock=*/0);
 
     if (!RPY_RDB_REPLAY) {
         rpy_reverse_db_flush();
@@ -137,6 +148,7 @@
             close(rpy_rev_fileno);
             rpy_rev_fileno = -1;
         }
+        _RPY_REVDB_UNLOCK();
     }
     else
         check_at_end(stop_points);
@@ -207,7 +219,7 @@
                     filename);
             abort();
         }
-        atexit(rpy_reverse_db_flush);
+        atexit(reverse_db_lock_and_flush);
 
         write_all(RDB_SIGNATURE, strlen(RDB_SIGNATURE));
         for (i = 0; i < argc; i++) {
@@ -244,8 +256,12 @@
 
 static void flush_buffer(void)
 {
+    /* must be called with the lock held */
+    ssize_t full_size;
+    assert(rpy_revdb.lock);
+
     /* write the current buffer content to the OS */
-    ssize_t full_size = rpy_revdb.buf_p - rpy_rev_buffer;
+    full_size = rpy_revdb.buf_p - rpy_rev_buffer;
     rpy_revdb.buf_p = rpy_rev_buffer + sizeof(int16_t);
     if (rpy_rev_fileno >= 0)
         write_all(rpy_rev_buffer, full_size);
@@ -253,13 +269,18 @@
 
 static ssize_t current_packet_size(void)
 {
+    /* must be called with the lock held */
     return rpy_revdb.buf_p - (rpy_rev_buffer + sizeof(int16_t));
 }
 
 RPY_EXTERN
 void rpy_reverse_db_flush(void)
 {
-    ssize_t content_size = current_packet_size();
+    /* must be called with the lock held */
+    ssize_t content_size;
+    assert(rpy_revdb.lock);
+
+    content_size = current_packet_size();
     if (content_size != 0) {
         char *p = rpy_rev_buffer;
         assert(0 < content_size && content_size <= 32767);
@@ -268,6 +289,18 @@
     }
 }
 
+RPY_EXTERN
+void rpy_reverse_db_lock_acquire(void)
+{
+    while (1) {
+        if (rpy_revdb.lock == 0) {
+            if (pypy_lock_test_and_set(&rpy_revdb.lock, 1) == 0)
+                break;   /* done */
+        }
+        sched_yield();
+    }
+}
+
 void boehm_gc_finalizer_notifier(void)
 {
     /* This is called by Boehm when there are pending finalizers.
@@ -303,6 +336,7 @@
     int64_t done;
 
     /* Write an ASYNC_FINALIZER_TRIGGER packet */
+    _RPY_REVDB_LOCK();
     rpy_reverse_db_flush();
     assert(current_packet_size() == 0);
 
@@ -310,6 +344,7 @@
     memcpy(rpy_revdb.buf_p, &rpy_revdb.stop_point_seen, sizeof(uint64_t));
     rpy_revdb.buf_p += sizeof(uint64_t);
     flush_buffer();
+    _RPY_REVDB_UNLOCK();
 
     /* Invoke all Boehm finalizers.  For new-style finalizers, this
        will only cause them to move to the queues, where
@@ -364,8 +399,10 @@
 
 static uint64_t recording_offset(void)
 {
+    /* must be called with the lock held */
     off_t base_offset;
     ssize_t extra_size = rpy_revdb.buf_p - rpy_rev_buffer;
+    assert(rpy_revdb.lock);
 
     if (rpy_rev_fileno < 0)
         return 1;
@@ -379,7 +416,10 @@
 
 static void patch_prev_offset(int64_t offset, char old, char new)
 {
+    /* must be called with the lock held */
     off_t base_offset;
+    assert(rpy_revdb.lock);
+
     if (rpy_rev_fileno < 0)
         return;
     base_offset = lseek(rpy_rev_fileno, 0, SEEK_CUR);
@@ -452,14 +492,18 @@
         /* Emit WEAKREF_AFTERWARDS_DEAD, but remember where we emit it.
            If we deref the weakref and it is still alive, we will patch
            it with WEAKREF_AFTERWARDS_ALIVE. */
-        if (!RPY_RDB_REPLAY)
+        if (!RPY_RDB_REPLAY) {
+            _RPY_REVDB_LOCK();
             r->re_off_prev = recording_offset();
+        }
         else
             r->re_off_prev = 1;    /* any number > 0 */
 
-        RPY_REVDB_EMIT(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive);
+        _RPY_REVDB_EMIT_L(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive,
+                          /*must_lock=*/0);
 
         if (!RPY_RDB_REPLAY) {
+            _RPY_REVDB_UNLOCK();
             OP_BOEHM_DISAPPEARING_LINK(&r->re_addr, target, /*nothing*/);
         }
         else {
@@ -498,13 +542,18 @@
         else {
             char alive;
             if (!RPY_RDB_REPLAY) {
+                _RPY_REVDB_LOCK();
                 patch_prev_offset(r->re_off_prev, WEAKREF_AFTERWARDS_DEAD,
                                                   WEAKREF_AFTERWARDS_ALIVE);
                 r->re_off_prev = recording_offset();
             }
-            RPY_REVDB_EMIT(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive);
+            _RPY_REVDB_EMIT_L(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive,
+                              /*must_lock=*/0);
 
-            if (RPY_RDB_REPLAY) {
+            if (!RPY_RDB_REPLAY) {
+                _RPY_REVDB_UNLOCK();
+            }
+            else {
                 switch (alive) {
                 case WEAKREF_AFTERWARDS_DEAD:
                     r->re_addr = NULL;
@@ -527,8 +576,10 @@
     locnum += 300;
     assert(locnum < 0xFC00);
     if (!RPY_RDB_REPLAY) {
-        _RPY_REVDB_EMIT_RECORD(unsigned char _e, (locnum >> 8));
-        _RPY_REVDB_EMIT_RECORD(unsigned char _e, (locnum & 0xFF));
+        _RPY_REVDB_LOCK();
+        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, (locnum >> 8));
+        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, (locnum & 0xFF));
+        _RPY_REVDB_UNLOCK();
     }
 }
 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -1,21 +1,21 @@
 #include <string.h>
+#include "src/thread.h"
 
-/* By default, this makes an executable which supports both recording
-   and replaying.  It should help avoid troubles like using for
-   replaying an executable that is slightly different than the one
-   used for recording.  In theory you can compile with
-   -DRPY_RDB_REPLAY=0 or -DRPY_RDB_REPLAY=1 to get only one version
-   compiled for it, which should be slightly faster (not tested so
-   far).
-*/
+/************************************************************
+ ***  RevDB --- record and replay debugging               ***
+ ************************************************************/
+
 
 typedef struct {
 #ifndef RPY_RDB_REPLAY
     bool_t replay;
 #define RPY_RDB_REPLAY   rpy_revdb.replay
 #define RPY_RDB_DYNAMIC_REPLAY
+#else
+# error "explicit RPY_RDB_REPLAY: not really supported"
 #endif
     bool_t watch_enabled;
+    long lock;
     char *buf_p, *buf_limit, *buf_readend;
     uint64_t stop_point_seen, stop_point_break;
     uint64_t unique_id_seen, unique_id_break;
@@ -59,7 +59,19 @@
 #endif
 
 
-#define _RPY_REVDB_EMIT_RECORD(decl_e, variable)                        \
+/* Acquire/release the lock around EMIT_RECORD, because it may be
+   called without holding the GIL.  Note that we're always
+   single-threaded during replaying: the lock is only useful during
+   recording. */
+#define _RPY_REVDB_LOCK()                                               \
+    if (pypy_lock_test_and_set(&rpy_revdb.lock, 1) != 0)                \
+        rpy_reverse_db_lock_acquire();
+
+#define _RPY_REVDB_UNLOCK()                                             \
+    pypy_lock_release(&rpy_revdb.lock)
+
+
+#define _RPY_REVDB_EMIT_RECORD_L(decl_e, variable)                      \
         {                                                               \
             decl_e = variable;                                          \
             _RPY_REVDB_PRINT("write", _e);                              \
@@ -81,21 +93,28 @@
             variable = _e;                                              \
         }
 
-#define RPY_REVDB_EMIT(normal_code, decl_e, variable)                   \
+#define _RPY_REVDB_EMIT_L(normal_code, decl_e, variable, must_lock)     \
     if (!RPY_RDB_REPLAY) {                                              \
         normal_code                                                     \
-        _RPY_REVDB_EMIT_RECORD(decl_e, variable)                        \
+        if (must_lock) _RPY_REVDB_LOCK();                               \
+        _RPY_REVDB_EMIT_RECORD_L(decl_e, variable)                      \
+        if (must_lock) _RPY_REVDB_UNLOCK();                             \
     } else                                                              \
         _RPY_REVDB_EMIT_REPLAY(decl_e, variable)
 
+#define RPY_REVDB_EMIT(normal_code, decl_e, variable)                   \
+    _RPY_REVDB_EMIT_L(normal_code, decl_e, variable, 1)
+
 #define RPY_REVDB_EMIT_VOID(normal_code)                                \
     if (!RPY_RDB_REPLAY) { normal_code } else { }
 
 #define RPY_REVDB_CALL(call_code, decl_e, variable)                     \
     if (!RPY_RDB_REPLAY) {                                              \
         call_code                                                       \
-        _RPY_REVDB_EMIT_RECORD(unsigned char _e, 0xFC)                  \
-        _RPY_REVDB_EMIT_RECORD(decl_e, variable)                        \
+        _RPY_REVDB_LOCK();                                              \
+        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, 0xFC)                \
+        _RPY_REVDB_EMIT_RECORD_L(decl_e, variable)                      \
+        _RPY_REVDB_UNLOCK();                                            \
     } else {                                                            \
         unsigned char _re;                                              \
         _RPY_REVDB_EMIT_REPLAY(unsigned char _e, _re)                   \
@@ -107,7 +126,9 @@
 #define RPY_REVDB_CALL_VOID(call_code)                                  \
     if (!RPY_RDB_REPLAY) {                                              \
         call_code                                                       \
-        _RPY_REVDB_EMIT_RECORD(unsigned char _e, 0xFC)                  \
+        _RPY_REVDB_LOCK();                                              \
+        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, 0xFC)                \
+        _RPY_REVDB_UNLOCK();                                            \
     }                                                                   \
     else {                                                              \
         unsigned char _re;                                              \
@@ -174,7 +195,7 @@
 #define RPY_REVDB_CAST_PTR_TO_INT(obj)   (((struct pypy_header0 *)obj)->h_uid)
 
 
-RPY_EXTERN void rpy_reverse_db_flush(void);
+RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);
 RPY_EXTERN void rpy_reverse_db_stop_point(long place);
 RPY_EXTERN void rpy_reverse_db_send_answer(int cmd, int64_t arg1, int64_t arg2,
@@ -193,5 +214,6 @@
 RPY_EXTERN void rpy_reverse_db_call_destructor(void *obj);
 RPY_EXTERN void rpy_reverse_db_invoke_callback(unsigned char);
 RPY_EXTERN void rpy_reverse_db_callback_loc(int);
+RPY_EXTERN void rpy_reverse_db_lock_acquire(void);
 
 /* ------------------------------------------------------------ */

From pypy.commits at gmail.com  Tue Aug  9 10:52:34 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 07:52:34 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Fix test_dictproxy.py to
 actually match the expected behaviour
Message-ID: <57a9ee32.c1e31c0a.c1830.168b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86108:50212a6235fd
Date: 2016-08-03 04:59 +0100
http://bitbucket.org/pypy/pypy/changeset/50212a6235fd/

Log:	Fix test_dictproxy.py to actually match the expected behaviour

diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -9,37 +9,19 @@
         assert 'a' in NotEmpty.__dict__
         assert 'a' in NotEmpty.__dict__.keys()
         assert 'b' not in NotEmpty.__dict__
-        NotEmpty.__dict__['b'] = 4
-        assert NotEmpty.b == 4
-        del NotEmpty.__dict__['b']
         assert NotEmpty.__dict__.get("b") is None
+        raises(TypeError, "NotEmpty.__dict__['b'] = 4")
         raises(TypeError, 'NotEmpty.__dict__[15] = "y"')
-        raises(KeyError, 'del NotEmpty.__dict__[15]')
+        raises(TypeError, 'del NotEmpty.__dict__[15]')
 
-        assert NotEmpty.__dict__.setdefault("string", 1) == 1
-        assert NotEmpty.__dict__.setdefault("string", 2) == 1
-        assert NotEmpty.string == 1
-        raises(TypeError, 'NotEmpty.__dict__.setdefault(15, 1)')
-
-    def test_dictproxy_popitem(self):
-        class A(object):
-            a = 42
-        seen = 0
-        try:
-            while True:
-                key, value = A.__dict__.popitem()
-                if key == 'a':
-                    assert value == 42
-                    seen += 1
-        except KeyError:
-            pass
-        assert seen == 1
+        raises(AttributeError, 'NotEmpty.__dict__.setdefault')
 
     def test_dictproxy_getitem(self):
         class NotEmpty(object):
             a = 1
         assert 'a' in NotEmpty.__dict__
-        class substr(str): pass
+        class substr(str):
+            pass
         assert substr('a') in NotEmpty.__dict__
         assert u'a' in NotEmpty.__dict__
         assert NotEmpty.__dict__[u'a'] == 1
@@ -62,15 +44,37 @@
         class a(object):
             pass
         s1 = repr(a.__dict__)
+        assert s1.startswith('dict_proxy({') and s1.endswith('})')
         s2 = str(a.__dict__)
-        assert s1 == s2
-        assert s1.startswith('{') and s1.endswith('}')
+        assert s1 == 'dict_proxy(%s)' % s2
 
     def test_immutable_dict_on_builtin_type(self):
         raises(TypeError, "int.__dict__['a'] = 1")
-        raises(TypeError, int.__dict__.popitem)
-        raises(TypeError, int.__dict__.clear)
+        raises((AttributeError, TypeError), "int.__dict__.popitem()")
+        raises((AttributeError, TypeError), "int.__dict__.clear()")
+
+    def test_dictproxy(self):
+        dictproxy = type(int.__dict__)
+        assert dictproxy is not dict
+        assert dictproxy.__name__ == 'dictproxy'
+        raises(TypeError, dictproxy)
+
+        mapping = {'a': 1}
+        raises(TypeError, dictproxy, mapping)
+
+        class A(object):
+            a = 1
+
+        proxy = A.__dict__
+        mapping = dict(proxy)
+        assert proxy['a'] == 1
+        assert 'a' in proxy
+        assert 'z' not in proxy
+        assert repr(proxy) == 'dict_proxy(%r)' % mapping
+        assert proxy.keys() == mapping.keys()
+        raises(TypeError, "proxy['a'] = 4")
+        raises(TypeError, "del proxy['a']")
+        raises(AttributeError, "proxy.clear()")
 
 class AppTestUserObjectMethodCache(AppTestUserObject):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
-

From pypy.commits at gmail.com  Tue Aug  9 10:52:36 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 07:52:36 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Rename DictProxyStrategy to
 ClassDictStrategy and dictproxyobject.py to classdict.py
Message-ID: <57a9ee34.2624c20a.7a5a1.45ba@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86109:82c4a72260a7
Date: 2016-08-02 20:01 +0100
http://bitbucket.org/pypy/pypy/changeset/82c4a72260a7/

Log:	Rename DictProxyStrategy to ClassDictStrategy and dictproxyobject.py
	to classdict.py

diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/classdict.py
rename from pypy/objspace/std/dictproxyobject.py
rename to pypy/objspace/std/classdict.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/classdict.py
@@ -7,7 +7,7 @@
 from pypy.objspace.std.typeobject import unwrap_cell
 
 
-class DictProxyStrategy(DictStrategy):
+class ClassDictStrategy(DictStrategy):
     erase, unerase = rerased.new_erasing_pair("dictproxy")
     erase = staticmethod(erase)
     unerase = staticmethod(unerase)
@@ -112,4 +112,4 @@
     def wrapvalue(space, value):
         return unwrap_cell(space, value)
 
-create_iterator_classes(DictProxyStrategy)
+create_iterator_classes(ClassDictStrategy)
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -486,11 +486,11 @@
             del self.lazyloaders
 
     def getdict(self, space): # returning a dict-proxy!
-        from pypy.objspace.std.dictproxyobject import DictProxyStrategy
+        from pypy.objspace.std.classdict import ClassDictStrategy
         from pypy.objspace.std.dictmultiobject import W_DictObject
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        strategy = space.fromcache(DictProxyStrategy)
+        strategy = space.fromcache(ClassDictStrategy)
         storage = strategy.erase(self)
         return W_DictObject(space, strategy, storage)
 

From pypy.commits at gmail.com  Tue Aug  9 10:52:39 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 07:52:39 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Do not attempt to modify a
 class dict, use setattr instead
Message-ID: <57a9ee37.cb7f1c0a.705ce.3d11@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86111:d10201ca7dc8
Date: 2016-08-04 16:29 +0100
http://bitbucket.org/pypy/pypy/changeset/d10201ca7dc8/

Log:	Do not attempt to modify a class dict, use setattr instead

diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -175,7 +175,7 @@
          "__new__"      : make_new(class_name),
          }
     pycppclass = metacpp(class_name, _drop_cycles(bases), d)
- 
+
     # cache result early so that the class methods can find the class itself
     setattr(scope, final_class_name, pycppclass)
 
@@ -192,13 +192,10 @@
     for dm_name in cppclass.get_datamember_names():
         cppdm = cppclass.get_datamember(dm_name)
 
-        # here, setattr() can not be used, because a data member can shadow one in
-        # its base class, resulting in the __set__() of its base class being called
-        # by setattr(); so, store directly on the dictionary
-        pycppclass.__dict__[dm_name] = cppdm
+        setattr(pycppclass, dm_name, cppdm)
         import cppyy
         if cppyy._is_static(cppdm):     # TODO: make this a method of cppdm
-            metacpp.__dict__[dm_name] = cppdm
+            setattr(metacpp, dm_name, cppdm)
 
     # the call to register will add back-end specific pythonizations and thus
     # needs to run first, so that the generic pythonizations can use them
@@ -413,7 +410,7 @@
         lib = cppyy._load_dictionary(name)
         _loaded_dictionaries[name] = lib
         return lib
-    
+
 def _init_pythonify():
     # cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause cppyy to be loaded

From pypy.commits at gmail.com  Tue Aug  9 10:52:41 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 07:52:41 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Add failing cpyext test
 clarifying the expected behaviour of type->tp_dict
Message-ID: <57a9ee39.c3881c0a.4cd4f.464f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86112:b2d8c440375b
Date: 2016-08-04 18:28 +0100
http://bitbucket.org/pypy/pypy/changeset/b2d8c440375b/

Log:	Add failing cpyext test clarifying the expected behaviour of
	type->tp_dict

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -282,11 +282,23 @@
                      args->ob_type->tp_dict, "copy");
                  Py_INCREF(method);
                  return method;
+             '''),
+            ("get_type_dict", "METH_O",
              '''
-             )
+                PyObject* value = args->ob_type->tp_dict;
+                if (value == NULL) value = Py_None;
+                Py_INCREF(value);
+                return value;
+             '''),
             ])
         obj = foo.new()
         assert module.read_tp_dict(obj) == foo.fooType.copy
+        assert type(module.get_type_dict(obj)) is dict
+        d = module.get_type_dict(1)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert int._some_attribute == 1
+        del d["_some_attribute"]
 
     def test_custom_allocation(self):
         foo = self.import_module("foo")

From pypy.commits at gmail.com  Tue Aug  9 10:52:37 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 07:52:37 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Backport W_DictProxyObject
 from py3k
Message-ID: <57a9ee35.17a71c0a.822ba.3f84@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86110:786fadddf616
Date: 2016-08-09 15:15 +0100
http://bitbucket.org/pypy/pypy/changeset/786fadddf616/

Log:	Backport W_DictProxyObject from py3k

diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -0,0 +1,86 @@
+# Read-only proxy for mappings. PyPy does not have a separate type for
+# type.__dict__, so PyDictProxy_New has to use a custom read-only mapping.
+
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
+from pypy.interpreter.typedef import TypeDef, interp2app
+
+class W_DictProxyObject(W_Root):
+    "Read-only proxy for mappings."
+
+    def __init__(self, w_mapping):
+        self.w_mapping = w_mapping
+
+    @staticmethod
+    def descr_new(space, w_type, w_mapping):
+        raise oefmt(space.w_TypeError, "Cannot create 'dictproxy' instances")
+
+    def descr_init(self, space, __args__):
+        pass
+
+    def descr_len(self, space):
+        return space.len(self.w_mapping)
+
+    def descr_getitem(self, space, w_key):
+        return space.getitem(self.w_mapping, w_key)
+
+    def descr_contains(self, space, w_key):
+        return space.contains(self.w_mapping, w_key)
+
+    def descr_iter(self, space):
+        return space.iter(self.w_mapping)
+
+    def descr_str(self, space):
+        return space.str(self.w_mapping)
+
+    def descr_repr(self, space):
+        return space.wrap("dict_proxy(%s)" %
+                                (space.str_w(space.repr(self.w_mapping)),))
+
+    @unwrap_spec(w_default=WrappedDefault(None))
+    def get_w(self, space, w_key, w_default):
+        return space.call_method(self.w_mapping, "get", w_key, w_default)
+
+    def keys_w(self, space):
+        return space.call_method(self.w_mapping, "keys")
+
+    def values_w(self, space):
+        return space.call_method(self.w_mapping, "values")
+
+    def items_w(self, space):
+        return space.call_method(self.w_mapping, "items")
+
+    def copy_w(self, space):
+        return space.call_method(self.w_mapping, "copy")
+
+cmp_methods = {}
+def make_cmp_method(op):
+    def descr_op(self, space, w_other):
+        return getattr(space, op)(self.w_mapping, w_other)
+    descr_name = 'descr_' + op
+    descr_op.__name__ = descr_name
+    setattr(W_DictProxyObject, descr_name, descr_op)
+    cmp_methods['__%s__' % op] = interp2app(getattr(W_DictProxyObject, descr_name))
+
+for op in ['eq', 'ne', 'gt', 'ge', 'lt', 'le']:
+    make_cmp_method(op)
+
+
+W_DictProxyObject.typedef = TypeDef(
+    'dictproxy',
+    __new__=interp2app(W_DictProxyObject.descr_new),
+    __init__=interp2app(W_DictProxyObject.descr_init),
+    __len__=interp2app(W_DictProxyObject.descr_len),
+    __getitem__=interp2app(W_DictProxyObject.descr_getitem),
+    __contains__=interp2app(W_DictProxyObject.descr_contains),
+    __iter__=interp2app(W_DictProxyObject.descr_iter),
+    __str__=interp2app(W_DictProxyObject.descr_str),
+    __repr__=interp2app(W_DictProxyObject.descr_repr),
+    get=interp2app(W_DictProxyObject.get_w),
+    keys=interp2app(W_DictProxyObject.keys_w),
+    values=interp2app(W_DictProxyObject.values_w),
+    items=interp2app(W_DictProxyObject.items_w),
+    copy=interp2app(W_DictProxyObject.copy_w),
+    **cmp_methods
+)
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -968,7 +968,6 @@
         raises(TypeError, setattr, list, 'foobar', 42)
         raises(TypeError, delattr, dict, 'keys')
         raises(TypeError, 'int.__dict__["a"] = 1')
-        raises(TypeError, 'int.__dict__.clear()')
 
     def test_nontype_in_mro(self):
         class OldStyle:
@@ -1026,10 +1025,9 @@
             pass
 
         a = A()
+        d = A.__dict__
         A.x = 1
-        assert A.__dict__["x"] == 1
-        A.__dict__['x'] = 5
-        assert A.x == 5
+        assert d["x"] == 1
 
     def test_we_already_got_one_1(self):
         # Issue #2079: highly obscure: CPython complains if we say
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -488,11 +488,13 @@
     def getdict(self, space): # returning a dict-proxy!
         from pypy.objspace.std.classdict import ClassDictStrategy
         from pypy.objspace.std.dictmultiobject import W_DictObject
+        from pypy.objspace.std.dictproxyobject import W_DictProxyObject
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
         strategy = space.fromcache(ClassDictStrategy)
         storage = strategy.erase(self)
-        return W_DictObject(space, strategy, storage)
+        w_dict = W_DictObject(space, strategy, storage)
+        return W_DictProxyObject(w_dict)
 
     def is_heaptype(self):
         return self.flag_heaptype

From pypy.commits at gmail.com  Tue Aug  9 10:52:47 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 07:52:47 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Cleanup
Message-ID: <57a9ee3f.c70a1c0a.38ae3.46ca@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86115:9d629ea0f59c
Date: 2016-08-08 18:45 +0100
http://bitbucket.org/pypy/pypy/changeset/9d629ea0f59c/

Log:	Cleanup

diff --git a/pypy/objspace/std/classdict.py b/pypy/objspace/std/classdict.py
--- a/pypy/objspace/std/classdict.py
+++ b/pypy/objspace/std/classdict.py
@@ -85,7 +85,8 @@
         return space.newlist_bytes(self.unerase(w_dict.dstorage).dict_w.keys())
 
     def values(self, w_dict):
-        return [unwrap_cell(self.space, w_value) for w_value in self.unerase(w_dict.dstorage).dict_w.itervalues()]
+        return [unwrap_cell(self.space, w_value) for w_value in
+                self.unerase(w_dict.dstorage).dict_w.itervalues()]
 
     def items(self, w_dict):
         space = self.space
@@ -103,12 +104,16 @@
 
     def getiterkeys(self, w_dict):
         return self.unerase(w_dict.dstorage).dict_w.iterkeys()
+
     def getitervalues(self, w_dict):
         return self.unerase(w_dict.dstorage).dict_w.itervalues()
+
     def getiteritems_with_hash(self, w_dict):
         return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w)
+
     def wrapkey(space, key):
         return space.wrap(key)
+
     def wrapvalue(space, value):
         return unwrap_cell(space, value)
 
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -1,5 +1,8 @@
-# Read-only proxy for mappings. PyPy does not have a separate type for
-# type.__dict__, so PyDictProxy_New has to use a custom read-only mapping.
+"""
+Read-only proxy for mappings.
+
+Its main use is as the return type of cls.__dict__.
+"""
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt

From pypy.commits at gmail.com  Tue Aug  9 10:52:43 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 07:52:43 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Make cls->tp_dict be a real
 dict while cls.__dict__ returns a proxy
Message-ID: <57a9ee3b.919a1c0a.5a71a.4b87@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86113:3555af06864f
Date: 2016-08-04 18:58 +0100
http://bitbucket.org/pypy/pypy/changeset/3555af06864f/

Log:	Make cls->tp_dict be a real dict while cls.__dict__ returns a proxy

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -293,12 +293,21 @@
             ])
         obj = foo.new()
         assert module.read_tp_dict(obj) == foo.fooType.copy
-        assert type(module.get_type_dict(obj)) is dict
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
         d = module.get_type_dict(1)
         assert type(d) is dict
-        d["_some_attribute"] = 1
-        assert int._some_attribute == 1
-        del d["_some_attribute"]
+        try:
+            d["_some_attribute"] = 1
+        except TypeError:  # on PyPy, int.__dict__ is really immutable
+            pass
+        else:
+            assert int._some_attribute == 1
+            del d["_some_attribute"]
 
     def test_custom_allocation(self):
         foo = self.import_module("foo")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -272,12 +272,12 @@
         if len(slot_names) == 1:
             if not getattr(pto, slot_names[0]):
                 setattr(pto, slot_names[0], slot_func_helper)
-        elif (w_type.getname(space) in ('list', 'tuple') and 
+        elif (w_type.getname(space) in ('list', 'tuple') and
               slot_names[0] == 'c_tp_as_number'):
             # XXX hack - hwo can we generalize this? The problem is method
             # names like __mul__ map to more than one slot, and we have no
             # convenient way to indicate which slots CPython have filled
-            # 
+            #
             # We need at least this special case since Numpy checks that
             # (list, tuple) do __not__ fill tp_as_number
             pass
@@ -860,8 +860,8 @@
 
     if w_obj.is_cpytype():
         Py_DecRef(space, pto.c_tp_dict)
-        w_dict = w_obj.getdict(space)
-        pto.c_tp_dict = make_ref(space, w_dict)
+    w_dict = w_obj.getdict(space)
+    pto.c_tp_dict = make_ref(space, w_dict)
 
 @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL)
 def PyType_IsSubtype(space, a, b):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -3,8 +3,8 @@
 from pypy.interpreter.baseobjspace import W_Root, SpaceCache
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import Function, StaticMethod
-from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\
-     descr_get_dict, dict_descr, Member, TypeDef
+from pypy.interpreter.typedef import (
+    weakref_descr, GetSetProperty, dict_descr, Member, TypeDef)
 from pypy.interpreter.astcompiler.misc import mangle
 from pypy.module.__builtin__ import abstractinst
 
@@ -485,16 +485,14 @@
                 self.getdictvalue(self.space, attr)
             del self.lazyloaders
 
-    def getdict(self, space): # returning a dict-proxy!
+    def getdict(self, space):
         from pypy.objspace.std.classdict import ClassDictStrategy
         from pypy.objspace.std.dictmultiobject import W_DictObject
-        from pypy.objspace.std.dictproxyobject import W_DictProxyObject
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
         strategy = space.fromcache(ClassDictStrategy)
         storage = strategy.erase(self)
-        w_dict = W_DictObject(space, strategy, storage)
-        return W_DictProxyObject(w_dict)
+        return W_DictObject(space, strategy, storage)
 
     def is_heaptype(self):
         return self.flag_heaptype
@@ -912,13 +910,20 @@
     return space.newbool(
         abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj))
 
+def type_get_dict(space, w_cls):
+    from pypy.objspace.std.dictproxyobject import W_DictProxyObject
+    w_dict = w_cls.getdict(space)
+    if w_dict is None:
+        return space.w_None
+    return W_DictProxyObject(w_dict)
+
 W_TypeObject.typedef = TypeDef("type",
     __new__ = gateway.interp2app(descr__new__),
     __name__ = GetSetProperty(descr_get__name__, descr_set__name__),
     __bases__ = GetSetProperty(descr_get__bases__, descr_set__bases__),
     __base__ = GetSetProperty(descr__base),
     __mro__ = GetSetProperty(descr_get__mro__),
-    __dict__ = GetSetProperty(descr_get_dict),
+    __dict__=GetSetProperty(type_get_dict),
     __doc__ = GetSetProperty(descr__doc),
     mro = gateway.interp2app(descr_mro),
     __flags__ = GetSetProperty(descr__flags),

From pypy.commits at gmail.com  Tue Aug  9 10:52:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 07:52:45 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Allow attribute deletion on
 C-defined types
Message-ID: <57a9ee3d.12331c0a.ea209.47ae@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86114:5cb67984053b
Date: 2016-08-05 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/5cb67984053b/

Log:	Allow attribute deletion on C-defined types

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -299,6 +299,15 @@
         assert type(obj)._some_attribute == 1
         del d["_some_attribute"]
 
+        class A(object):
+            pass
+        obj = A()
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
         d = module.get_type_dict(1)
         assert type(d) is dict
         try:
@@ -376,6 +385,21 @@
 
         api.Py_DecRef(ref)
 
+    def test_type_dict(self, space, api):
+        w_class = space.appexec([], """():
+            class A(object):
+                pass
+            return A
+            """)
+        ref = make_ref(space, w_class)
+
+        py_type = rffi.cast(PyTypeObjectPtr, ref)
+        w_dict = from_ref(space, py_type.c_tp_dict)
+        w_name = space.wrap('a')
+        space.setitem(w_dict, w_name, space.wrap(1))
+        assert space.int_w(space.getattr(w_class, w_name)) == 1
+        space.delitem(w_dict, w_name)
+
     def test_multiple_inheritance(self, space, api):
         w_class = space.appexec([], """():
             class A(object):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -346,7 +346,7 @@
     def deldictvalue(self, space, key):
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        if not self.is_heaptype():
+        if not (self.is_heaptype() or self.is_cpytype()):
             raise oefmt(space.w_TypeError,
                         "can't delete attributes on type object '%N'", self)
         try:

From pypy.commits at gmail.com  Tue Aug  9 11:38:55 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 09 Aug 2016 08:38:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async-translate: assert is instance Module,
 rpython deduces it is a W_Root
Message-ID: <57a9f90f.09afc20a.998b.433f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async-translate
Changeset: r86116:435335a96064
Date: 2016-08-09 17:38 +0200
http://bitbucket.org/pypy/pypy/changeset/435335a96064/

Log:	assert is instance Module, rpython deduces it is a W_Root properly
	wrap for load const

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1310,7 +1310,7 @@
                     nsubkwargs += 1
                 elif nsubkwargs:
                     # A keyword argument and we already have a dict.
-                    self.load_const(kw.arg)
+                    self.load_const(self.space.wrap(kw.arg.decode('utf-8')))
                     kw.value.walkabout(self)
                     nseen += 1
                 else:
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -85,6 +85,7 @@
         pathname = "<frozen %s>" % modulename
         code_w = ec.compiler.compile(source, pathname, 'exec', 0)
         w_mod = add_module(space, space.wrap(modulename))
+        assert isinstance(w_mod, Module) # XXX why is that necessary?
         space.setitem(space.sys.get('modules'), w_mod.w_name, w_mod)
         space.setitem(w_mod.w_dict, space.wrap('__name__'), w_mod.w_name)
         code_w.exec_code(space, w_mod.w_dict, w_mod.w_dict)

From pypy.commits at gmail.com  Tue Aug  9 11:53:55 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 09 Aug 2016 08:53:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Merge with py3.5-async-translate
Message-ID: <57a9fc93.a717c20a.155f5.591a@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86117:b7ea325def94
Date: 2016-08-09 17:53 +0200
http://bitbucket.org/pypy/pypy/changeset/b7ea325def94/

Log:	Merge with py3.5-async-translate

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1310,7 +1310,7 @@
                     nsubkwargs += 1
                 elif nsubkwargs:
                     # A keyword argument and we already have a dict.
-                    self.load_const(kw.arg)
+                    self.load_const(self.space.wrap(kw.arg.decode('utf-8')))
                     kw.value.walkabout(self)
                     nseen += 1
                 else:
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -581,12 +581,13 @@
         # position, then raise a GeneratorExit.  Otherwise, there is
         # no point.
         # If coroutine was never awaited on issue a RuntimeWarning.
-        if self.pycode is not None:
-            if self.frame is not None:
-                if self.frame.fget_f_lasti(self.frame).int_w(self.space) == -1:
-                    raise oefmt(space.w_RuntimeWarning,
-                                "coroutine '%s' was never awaited",
-                                self.pycode.co_name)
+        if self.pycode is not None and \
+           self.frame is not None and \
+           self.frame.last_instr == -1:
+            # XXX PyErr_Occured in condition?
+            raise oefmt(self.space.w_RuntimeWarning,
+                        "coroutine '%s' was never awaited",
+                        self.pycode.co_name)
         if self.frame is not None:
             block = self.frame.lastblock
             while block is not None:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -6,7 +6,7 @@
 
 from rpython.rlib import jit, rstackovf, rstring
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, always_inline
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -45,6 +45,26 @@
 
     return func_with_new_name(opimpl, "opcode_impl_for_%s" % operationname)
 
+def get_func_desc(space, func):
+    if isinstance(func,function.Function):
+        return "()"
+    elif isinstance(func, function.Method):
+        return "()"
+    else:
+        return " object";
+
+ at always_inline
+def list_unpack_helper(frame, itemcount):
+    space = frame.space
+    w_sum = space.newlist([], sizehint=itemcount)
+    for i in range(itemcount, 0, -1):
+        w_item = frame.peekvalue(i-1)
+        w_sum.extend(w_item)
+    while itemcount != 0:
+        frame.popvalue()
+        itemcount -= 1
+    return w_sum
+
 
 opcodedesc = bytecode_spec.opcodedesc
 HAVE_ARGUMENT = bytecode_spec.HAVE_ARGUMENT
@@ -1351,74 +1371,73 @@
             self.space.call_method(w_set, 'add', w_item)
         self.pushvalue(w_set)
 
-    def unpack_helper(self, itemcount, next_instr):
-        w_sum = []
+    def BUILD_SET_UNPACK(self, itemcount, next_instr):
+        space = self.space
+        w_sum = space.newset()
         for i in range(itemcount, 0, -1):
             w_item = self.peekvalue(i-1)
-            items = self.space.fixedview(w_item)
-            w_sum.extend(items)
+            # cannot use w_sum.update, w_item might not be a set
+            iterator = w_item.itervalues()
+            while True:
+                w_value = iterator.next_value()
+                if w_value is None:
+                    break
+                w_sum.add(w_value)
         while itemcount != 0:
             self.popvalue()
             itemcount -= 1
-        return w_sum
-
-    def BUILD_SET_UNPACK(self, itemcount, next_instr):
-        w_sum = self.unpack_helper(itemcount, next_instr)
-        self.pushvalue(self.space.newset(w_sum))
+        self.pushvalue(w_sum)
 
     def BUILD_TUPLE_UNPACK(self, itemcount, next_instr):
-        w_sum = self.unpack_helper(itemcount, next_instr)
-        self.pushvalue(self.space.newtuple(w_sum))
-        
+        w_list = list_unpack_helper(self, itemcount)
+        items = [w_obj for w_obj in w_list.getitems_unroll()]
+        self.pushvalue(self.space.newtuple(items))
+
     def BUILD_LIST_UNPACK(self, itemcount, next_instr):
-        w_sum = self.unpack_helper(itemcount, next_instr)
-        self.pushvalue(self.space.newlist(w_sum))
-        
-    def getFuncDesc(self, func):
-        if self.space.type(func).name.decode('utf-8') == 'method':
-            return "()"
-        elif self.space.type(func).name.decode('utf-8') == 'function':
-            return "()"
-        else:
-            return " object";
-    
+        w_sum = list_unpack_helper(self, itemcount)
+        self.pushvalue(w_sum)
+
     def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr):
+        space = self.space
         num_maps = itemcount & 0xff
         function_location = (itemcount>>8) & 0xff
-        w_dict = self.space.newdict()
-        dict_class = w_dict.__class__
+        w_dict = space.newdict()
         for i in range(num_maps, 0, -1):
             w_item = self.peekvalue(i-1)
-            if not issubclass(w_item.__class__, dict_class):
-                raise oefmt(self.space.w_TypeError,
+            if not space.ismapping_w(w_item):
+                raise oefmt(space.w_TypeError,
                         "'%T' object is not a mapping", w_item)
-            num_items = w_item.length()
-            keys = w_item.w_keys()
-            for j in range(num_items):
-                if self.space.type(keys.getitem(j)).name.decode('utf-8') == 'method':
+            iterator = w_item.iterkeys()
+            while True:
+                w_key = iterator.next_key()
+                if w_key is None:
+                    break
+                if not isinstance(w_key, space.UnicodeObjectCls):
                     err_fun = self.peekvalue(num_maps + function_location-1)
-                    raise oefmt(self.space.w_TypeError,
-                        "%N%s keywords must be strings", err_fun, getFuncDesc(err_fun))
-                if self.space.is_true(self.space.contains(w_dict,keys.getitem(j))):
+                    raise oefmt(space.w_TypeError,
+                        "%N%s keywords must be strings", err_fun,
+                                                         get_func_desc(space, err_fun))
+                if space.is_true(space.contains(w_dict,w_key)):
                     err_fun = self.peekvalue(num_maps + function_location-1)
-                    err_arg = self.space.unicode_w(keys.getitem(j))
-                    raise oefmt(self.space.w_TypeError,
-                        "%N%s got multiple values for keyword argument '%s'", err_fun, self.getFuncDesc(err_fun), err_arg)
-            self.space.call_method(w_dict, 'update', w_item)
+                    err_arg = w_key
+                    raise oefmt(space.w_TypeError,
+                        "%N%s got multiple values for keyword argument '%s'",
+                          err_fun, get_func_desc(space, err_fun), space.str_w(err_arg))
+            space.call_method(w_dict, 'update', w_item)
         while num_maps != 0:
             self.popvalue()
             num_maps -= 1
         self.pushvalue(w_dict)
-        
+
     def BUILD_MAP_UNPACK(self, itemcount, next_instr):
-        w_dict = self.space.newdict()
-        dict_class = w_dict.__class__
+        space = self.space
+        w_dict = space.newdict()
         for i in range(itemcount, 0, -1):
             w_item = self.peekvalue(i-1)
-            if not issubclass(w_item.__class__, dict_class):
+            if not space.ismapping_w(w_item):
                 raise oefmt(self.space.w_TypeError,
                         "'%T' object is not a mapping", w_item)
-            self.space.call_method(w_dict, 'update', w_item)
+            space.call_method(w_dict, 'update', w_item)
         while itemcount != 0:
             self.popvalue()
             itemcount -= 1
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -1,9 +1,13 @@
 class AppTestAsyncIO(object):
     
-    spaceconfig = dict(usemodules=["select","_socket","thread","signal","struct","_multiprocessing","array","_posixsubprocess","fcntl","unicodedata"])
+    spaceconfig = dict(usemodules=["select","_socket","thread","signal",
+                                   "struct","_multiprocessing","array",
+                                   "_posixsubprocess","fcntl",
+                                   "unicodedata"])
     
     def test_gil_issue(self):
-        # the problem occured at await asyncio.open_connection after calling run_until_complete
+        # the problem occured at await asyncio.open_connection
+        # after calling run_until_complete
         """
         import encodings.idna
         import asyncio
@@ -11,4 +15,6 @@
             reader, writer = await asyncio.open_connection('example.com', 80)
         
         loop = asyncio.get_event_loop()
-        loop.run_until_complete(f())"""
+        loop.run_until_complete(f())
+        print("done with async loop")
+        """
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -85,6 +85,7 @@
         pathname = "<frozen %s>" % modulename
         code_w = ec.compiler.compile(source, pathname, 'exec', 0)
         w_mod = add_module(space, space.wrap(modulename))
+        assert isinstance(w_mod, Module) # XXX why is that necessary?
         space.setitem(space.sys.get('modules'), w_mod.w_name, w_mod)
         space.setitem(w_mod.w_dict, space.wrap('__name__'), w_mod.w_name)
         code_w.exec_code(space, w_mod.w_dict, w_mod.w_dict)
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -82,7 +82,7 @@
                    for key in self.cache.keys()]
         return space.newlist(items_w)
 
-    def iterkeys(self, space):
+    def iteratekeys(self, space):
         return space.iter(self.keys(space))
 
     def itervalues(self, space):
@@ -106,11 +106,11 @@
     'zip_dict',
     __getitem__ = interp2app(W_ZipCache.getitem),
     __contains__ = interp2app(W_ZipCache.contains),
-    __iter__ = interp2app(W_ZipCache.iterkeys),
+    __iter__ = interp2app(W_ZipCache.iteratekeys),
     items = interp2app(W_ZipCache.items),
     iteritems = interp2app(W_ZipCache.iteritems),
     keys = interp2app(W_ZipCache.keys),
-    iterkeys = interp2app(W_ZipCache.iterkeys),
+    iterkeys = interp2app(W_ZipCache.iteratekeys),
     values = interp2app(W_ZipCache.values),
     itervalues = interp2app(W_ZipCache.itervalues),
     clear = interp2app(W_ZipCache.clear),
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -198,7 +198,6 @@
     
     def get_native_fmtchar(self, fmt):
         from rpython.rtyper.lltypesystem import rffi
-        from sys import getsizeof
         size = -1
         if fmt[0] == '@':
             f = fmt[1]
@@ -215,7 +214,7 @@
         elif f == 'q' or f == 'Q':
             size = rffi.sizeof(rffi.LONGLONG)
         elif f == 'n' or f == 'N':
-            size = getsizeof(rffi.r_ssize_t)
+            size = rffi.sizeof(rffi.SIZE_T)
         elif f == 'f':
             size = rffi.sizeof(rffi.FLOAT)
         elif f == 'd':
@@ -225,7 +224,7 @@
         elif f == 'P':
             size = rffi.sizeof(rffi.VOIDP)
         return size
-    
+
     def descr_cast(self, space, w_format, w_shape=None):
         # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
         self._check_released(space)

From pypy.commits at gmail.com  Tue Aug  9 12:05:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 09:05:55 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: in-progress
Message-ID: <57a9ff63.a719c20a.af26b.fdb1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86118:9a959ce7e35a
Date: 2016-08-09 18:05 +0200
http://bitbucket.org/pypy/pypy/changeset/9a959ce7e35a/

Log:	in-progress

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -284,7 +284,6 @@
                requires=[('translation.split_gc_address_space', True),
                          ('translation.jit', False),
                          ('translation.gc', 'boehm'),
-                         ('translation.thread', False),
                          ('translation.continuation', False)]),
 ])
 
diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -26,6 +26,10 @@
 
 #endif /* !_WIN32 */
 
+#ifdef RPY_REVERSE_DEBUGGER
+RPY_EXTERN void rpy_reverse_db_thread_switch(void);
+#endif
+
 RPY_EXTERN void RPyGilAllocate(void);
 RPY_EXTERN long RPyGilYieldThread(void);
 RPY_EXTERN void RPyGilAcquireSlowPath(long);
@@ -45,6 +49,9 @@
     long old_fastgil = pypy_lock_test_and_set(&rpy_fastgil, 1);
     if (old_fastgil != 0)
         RPyGilAcquireSlowPath(old_fastgil);
+#ifdef RPY_REVERSE_DEBUGGER
+    rpy_reverse_db_thread_switch();
+#endif
 }
 static inline void _RPyGilRelease(void) {
     assert(RPY_FASTGIL_LOCKED(rpy_fastgil));
diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py
--- a/rpython/translator/revdb/gencsupp.py
+++ b/rpython/translator/revdb/gencsupp.py
@@ -82,9 +82,10 @@
                '_revdb_do_all_calls_', False):
         return call_code   # a hack for ll_call_destructor() to mean
                            # that the calls should really be done
-    # haaaaack
+    #
+    # hack: we don't need the flag for at least these two common functions
     if call_code in ('RPyGilRelease();', 'RPyGilAcquire();'):
-        return '/* ' + call_code + ' */'
+        return 'RPY_REVDB_CALL_GILCTRL(%s);' % (call_code,)
     #
     tp = funcgen.lltypename(v_result)
     if tp == 'void @':
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -35,6 +35,7 @@
 #define WEAKREF_AFTERWARDS_ALIVE   ((char)0xeb)
 
 #define ASYNC_FINALIZER_TRIGGER    ((int16_t)0xff46)
+#define ASYNC_THREAD_SWITCH        ((int16_t)0xff54)
 
 #define FID_REGULAR_MODE           'R'
 #define FID_SAVED_STATE            'S'
@@ -55,6 +56,8 @@
 static char rpy_rev_buffer[16384];    /* max. 32768 */
 int rpy_rev_fileno = -1;
 static char flag_io_disabled = FID_REGULAR_MODE;
+static pthread_t current_logged_thread;
+static bool_t current_logged_thread_seen;
 
 
 static void setup_record_mode(int argc, char *argv[]);
@@ -321,6 +324,21 @@
 
 static long in_invoke_finalizers;
 
+static void emit_async_block(int async_code, uint64_t content)
+{
+    char *p = rpy_rev_buffer;
+
+    _RPY_REVDB_LOCK();
+    rpy_reverse_db_flush();
+    assert(current_packet_size() == 0);
+
+    *(int16_t *)p = async_code;
+    memcpy(rpy_revdb.buf_p, &content, sizeof(uint64_t));
+    rpy_revdb.buf_p += sizeof(uint64_t);
+    flush_buffer();
+    _RPY_REVDB_UNLOCK();
+}
+
 static void record_stop_point(void)
 {
     /* ===== FINALIZERS =====
@@ -332,19 +350,10 @@
        conceptually just *after* the stop point.
      */
     int i;
-    char *p = rpy_rev_buffer;
     int64_t done;
 
     /* Write an ASYNC_FINALIZER_TRIGGER packet */
-    _RPY_REVDB_LOCK();
-    rpy_reverse_db_flush();
-    assert(current_packet_size() == 0);
-
-    *(int16_t *)p = ASYNC_FINALIZER_TRIGGER;
-    memcpy(rpy_revdb.buf_p, &rpy_revdb.stop_point_seen, sizeof(uint64_t));
-    rpy_revdb.buf_p += sizeof(uint64_t);
-    flush_buffer();
-    _RPY_REVDB_UNLOCK();
+    emit_async_block(ASYNC_FINALIZER_TRIGGER, rpy_revdb.stop_point_seen);
 
     /* Invoke all Boehm finalizers.  For new-style finalizers, this
        will only cause them to move to the queues, where
@@ -365,6 +374,26 @@
 }
 
 RPY_EXTERN
+void rpy_reverse_db_thread_switch(void)
+{
+    /* called at the end of _RPyGilAcquire(), when there was
+       potentially a thread switch.  If there actually was, emit an
+       ASYNC_THREAD_SWITCH block. */
+    pthread_t tself;
+    assert(!RPY_RDB_REPLAY);
+
+    tself = pthread_self();
+    if (!current_logged_thread_seen) {
+        current_logged_thread = tself;
+        current_logged_thread_seen = 1;
+    }
+    else if (!pthread_equal(tself, current_logged_thread)) {
+        emit_async_block(ASYNC_THREAD_SWITCH, (uint64_t)tself);
+        current_logged_thread = tself;
+    }
+}
+
+RPY_EXTERN
 void rpy_reverse_db_call_destructor(void *obj)
 {
     /* old-style finalizers.  Should occur only from the 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -137,6 +137,11 @@
             rpy_reverse_db_invoke_callback(_re);                        \
     }
 
+#define RPY_REVDB_CALL_GILCTRL(call_code)                               \
+    if (!RPY_RDB_REPLAY) {                                              \
+        call_code                                                       \
+    }
+
 #define RPY_REVDB_CALLBACKLOC(locnum)                                   \
     rpy_reverse_db_callback_loc(locnum)
 
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -86,13 +86,13 @@
         return self.cur == len(self.buffer)
 
     def write_call(self, expected_string):
-        x = self.next()   # raw_malloc: the pointer we got
-        self.same_thread()
+        x = self.next()     # raw_malloc: the pointer we got
+        self.same_stack()   # write
         x = self.next(); assert x == len(expected_string)
-        self.same_thread()
+        self.same_stack()   # errno
         x = self.next('i'); assert x == 0      # errno
 
-    def same_thread(self):
+    def same_stack(self):
         x = self.next('c'); assert x == '\xFC'
 
 
diff --git a/rpython/translator/revdb/test/test_callback.py b/rpython/translator/revdb/test/test_callback.py
--- a/rpython/translator/revdb/test/test_callback.py
+++ b/rpython/translator/revdb/test/test_callback.py
@@ -63,14 +63,14 @@
         self.compile(main, backendopt=False)
         out = self.run('Xx')
         rdb = self.fetch_rdb([self.exename, 'Xx'])
-        rdb.same_thread()                       # callmesimple()
+        rdb.same_stack()                        # callmesimple()
         x = rdb.next('i'); assert x == 55555
         rdb.write_call('55555\n')
         b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
         x = rdb.next('i'); assert x == 40       # arg n
         x = rdb.next('!h'); assert x == b       # -> callback
         x = rdb.next('i'); assert x == 3        # arg n
-        rdb.same_thread()                       # <- return in main thread
+        rdb.same_stack()                        # <- return in main thread
         x = rdb.next('i'); assert x == 4000 * 300   # return from callme()
         rdb.write_call('%s\n' % (4000 * 300,))
         x = rdb.next('q'); assert x == 0      # number of stop points
@@ -87,7 +87,7 @@
         x = rdb.next('!h'); assert x == b       # -> callback again
         x = rdb.next('i'); assert x == 3        # arg n
         rdb.write_call('3\n')
-        rdb.same_thread()                       # -> return in main thread
+        rdb.same_stack()                        # -> return in main thread
         x = rdb.next('i'); assert x == 120      # <- return from callme()
         rdb.write_call('120\n')
         x = rdb.next('q'); assert x == 2        # number of stop points
diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py
--- a/rpython/translator/revdb/test/test_weak.py
+++ b/rpython/translator/revdb/test/test_weak.py
@@ -203,7 +203,7 @@
                 assert time == i + 1
                 y = intmask(rdb.next('q')); assert y == -1
                 triggered = True
-            rdb.same_thread()
+            rdb.same_stack()
             j = rdb.next()
             assert j == i + 1000000 * triggered
             if triggered:
@@ -215,7 +215,7 @@
                     assert uid > 0 and uid not in uid_seen
                     uid_seen.add(uid)
                     lst.append(uid)
-                rdb.same_thread()
+                rdb.same_stack()
                 totals.append((lst, intmask(rdb.next())))
         x = rdb.next('q'); assert x == 3000    # number of stop points
         #
@@ -245,13 +245,13 @@
                     assert x != -1
                     assert x not in seen_uids
                     seen_uids.add(x)
-                    rdb.same_thread()
+                    rdb.same_stack()
                     y = intmask(rdb.next())
                     assert y == -7      # from the __del__
                     x = intmask(rdb.next())
                     if x == -1:
                         break
-            rdb.same_thread()
+            rdb.same_stack()
             x = rdb.next()
             assert x == len(seen_uids)
         assert len(seen_uids) == int(out)

From pypy.commits at gmail.com  Tue Aug  9 12:55:01 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 09:55:01 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Add typechecking to
 type.__dict__ descriptor
Message-ID: <57aa0ae5.2916c20a.891ef.6245@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86119:24dfaf3c5ca1
Date: 2016-08-09 17:54 +0100
http://bitbucket.org/pypy/pypy/changeset/24dfaf3c5ca1/

Log:	Add typechecking to type.__dict__ descriptor

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -911,6 +911,7 @@
         abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj))
 
 def type_get_dict(space, w_cls):
+    w_cls = _check(space, w_cls)
     from pypy.objspace.std.dictproxyobject import W_DictProxyObject
     w_dict = w_cls.getdict(space)
     if w_dict is None:

From pypy.commits at gmail.com  Tue Aug  9 13:12:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 10:12:20 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: still in-progress: write the
 ASYNC_THREAD_SWITCH blocks just
Message-ID: <57aa0ef4.85c11c0a.6406c.759b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86120:46485b2edf2b
Date: 2016-08-09 19:11 +0200
http://bitbucket.org/pypy/pypy/changeset/46485b2edf2b/

Log:	still in-progress: write the ASYNC_THREAD_SWITCH blocks just before
	we write other things to the log

diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -26,10 +26,6 @@
 
 #endif /* !_WIN32 */
 
-#ifdef RPY_REVERSE_DEBUGGER
-RPY_EXTERN void rpy_reverse_db_thread_switch(void);
-#endif
-
 RPY_EXTERN void RPyGilAllocate(void);
 RPY_EXTERN long RPyGilYieldThread(void);
 RPY_EXTERN void RPyGilAcquireSlowPath(long);
@@ -49,9 +45,6 @@
     long old_fastgil = pypy_lock_test_and_set(&rpy_fastgil, 1);
     if (old_fastgil != 0)
         RPyGilAcquireSlowPath(old_fastgil);
-#ifdef RPY_REVERSE_DEBUGGER
-    rpy_reverse_db_thread_switch();
-#endif
 }
 static inline void _RPyGilRelease(void) {
     assert(RPY_FASTGIL_LOCKED(rpy_fastgil));
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -56,8 +56,8 @@
 static char rpy_rev_buffer[16384];    /* max. 32768 */
 int rpy_rev_fileno = -1;
 static char flag_io_disabled = FID_REGULAR_MODE;
-static pthread_t current_logged_thread;
-static bool_t current_logged_thread_seen;
+__thread bool_t rpy_active_thread;
+static bool_t *rpy_active_thread_ptr;
 
 
 static void setup_record_mode(int argc, char *argv[]);
@@ -254,6 +254,9 @@
     rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32;
     rpy_revdb.unique_id_seen = 1;
 
+    rpy_active_thread = 1;
+    rpy_active_thread_ptr = &rpy_active_thread;
+
     pthread_atfork(NULL, NULL, close_revdb_fileno_in_fork_child);
 }
 
@@ -292,18 +295,6 @@
     }
 }
 
-RPY_EXTERN
-void rpy_reverse_db_lock_acquire(void)
-{
-    while (1) {
-        if (rpy_revdb.lock == 0) {
-            if (pypy_lock_test_and_set(&rpy_revdb.lock, 1) == 0)
-                break;   /* done */
-        }
-        sched_yield();
-    }
-}
-
 void boehm_gc_finalizer_notifier(void)
 {
     /* This is called by Boehm when there are pending finalizers.
@@ -339,6 +330,24 @@
     _RPY_REVDB_UNLOCK();
 }
 
+RPY_EXTERN
+void rpy_reverse_db_lock_acquire(void)
+{
+    assert(!RPY_RDB_REPLAY);
+    while (1) {
+        if (rpy_revdb.lock == 0) {
+            if (pypy_lock_test_and_set(&rpy_revdb.lock, 1) == 0)
+                break;   /* done */
+        }
+        sched_yield();
+    }
+    /* we have acquired the lock here */
+    *rpy_active_thread_ptr = 0;
+    rpy_active_thread = 1;
+    rpy_active_thread_ptr = &rpy_active_thread;
+    emit_async_block(ASYNC_THREAD_SWITCH, (uint64_t)pthread_self());
+}
+
 static void record_stop_point(void)
 {
     /* ===== FINALIZERS =====
@@ -374,26 +383,6 @@
 }
 
 RPY_EXTERN
-void rpy_reverse_db_thread_switch(void)
-{
-    /* called at the end of _RPyGilAcquire(), when there was
-       potentially a thread switch.  If there actually was, emit an
-       ASYNC_THREAD_SWITCH block. */
-    pthread_t tself;
-    assert(!RPY_RDB_REPLAY);
-
-    tself = pthread_self();
-    if (!current_logged_thread_seen) {
-        current_logged_thread = tself;
-        current_logged_thread_seen = 1;
-    }
-    else if (!pthread_equal(tself, current_logged_thread)) {
-        emit_async_block(ASYNC_THREAD_SWITCH, (uint64_t)tself);
-        current_logged_thread = tself;
-    }
-}
-
-RPY_EXTERN
 void rpy_reverse_db_call_destructor(void *obj)
 {
     /* old-style finalizers.  Should occur only from the 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -23,6 +23,7 @@
 
 RPY_EXTERN rpy_revdb_t rpy_revdb;
 RPY_EXTERN int rpy_rev_fileno;
+RPY_EXTERN __thread bool_t rpy_active_thread;
 
 
 /* ------------------------------------------------------------ */
@@ -64,7 +65,8 @@
    single-threaded during replaying: the lock is only useful during
    recording. */
 #define _RPY_REVDB_LOCK()                                               \
-    if (pypy_lock_test_and_set(&rpy_revdb.lock, 1) != 0)                \
+    if (!rpy_active_thread ||                                           \
+            pypy_lock_test_and_set(&rpy_revdb.lock, 1) != 0)            \
         rpy_reverse_db_lock_acquire();
 
 #define _RPY_REVDB_UNLOCK()                                             \

From pypy.commits at gmail.com  Tue Aug  9 13:14:44 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 10:14:44 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Implement methods: iterkeys(),
 itervalues(), iteritems().
Message-ID: <57aa0f84.ca11c30a.76936.64ff@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86121:d7ad5289b8ff
Date: 2016-08-09 18:14 +0100
http://bitbucket.org/pypy/pypy/changeset/d7ad5289b8ff/

Log:	Implement methods: iterkeys(), itervalues(), iteritems().

diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -48,12 +48,21 @@
     def keys_w(self, space):
         return space.call_method(self.w_mapping, "keys")
 
+    def descr_iterkeys(self, space):
+        return space.call_method(self.w_mapping, "iterkeys")
+
     def values_w(self, space):
         return space.call_method(self.w_mapping, "values")
 
+    def descr_itervalues(self, space):
+        return space.call_method(self.w_mapping, "itervalues")
+
     def items_w(self, space):
         return space.call_method(self.w_mapping, "items")
 
+    def descr_iteritems(self, space):
+        return space.call_method(self.w_mapping, "iteritems")
+
     def copy_w(self, space):
         return space.call_method(self.w_mapping, "copy")
 
@@ -82,8 +91,11 @@
     __repr__=interp2app(W_DictProxyObject.descr_repr),
     get=interp2app(W_DictProxyObject.get_w),
     keys=interp2app(W_DictProxyObject.keys_w),
+    iterkeys=interp2app(W_DictProxyObject.descr_iterkeys),
     values=interp2app(W_DictProxyObject.values_w),
+    itervalues=interp2app(W_DictProxyObject.descr_itervalues),
     items=interp2app(W_DictProxyObject.items_w),
+    iteritems=interp2app(W_DictProxyObject.descr_iteritems),
     copy=interp2app(W_DictProxyObject.copy_w),
     **cmp_methods
 )
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -72,6 +72,9 @@
         assert 'z' not in proxy
         assert repr(proxy) == 'dict_proxy(%r)' % mapping
         assert proxy.keys() == mapping.keys()
+        assert list(proxy.iterkeys()) == proxy.keys()
+        assert list(proxy.itervalues()) == proxy.values()
+        assert list(proxy.iteritems()) == proxy.items()
         raises(TypeError, "proxy['a'] = 4")
         raises(TypeError, "del proxy['a']")
         raises(AttributeError, "proxy.clear()")

From pypy.commits at gmail.com  Tue Aug  9 16:42:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 13:42:07 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: recording threads
Message-ID: <57aa401f.c75dc20a.63be0.0d25@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86122:738711fb525a
Date: 2016-08-09 22:41 +0200
http://bitbucket.org/pypy/pypy/changeset/738711fb525a/

Log:	recording threads

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -254,7 +254,7 @@
     rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32;
     rpy_revdb.unique_id_seen = 1;
 
-    rpy_active_thread = 1;
+    rpy_active_thread = 0;  /* write an ASYNC_THREAD_SWITCH first in the log */
     rpy_active_thread_ptr = &rpy_active_thread;
 
     pthread_atfork(NULL, NULL, close_revdb_fileno_in_fork_child);
@@ -317,9 +317,10 @@
 
 static void emit_async_block(int async_code, uint64_t content)
 {
+    /* must be called with the lock held */
     char *p = rpy_rev_buffer;
+    assert(rpy_revdb.lock);
 
-    _RPY_REVDB_LOCK();
     rpy_reverse_db_flush();
     assert(current_packet_size() == 0);
 
@@ -327,12 +328,12 @@
     memcpy(rpy_revdb.buf_p, &content, sizeof(uint64_t));
     rpy_revdb.buf_p += sizeof(uint64_t);
     flush_buffer();
-    _RPY_REVDB_UNLOCK();
 }
 
 RPY_EXTERN
 void rpy_reverse_db_lock_acquire(void)
 {
+    uint64_t pself;
     assert(!RPY_RDB_REPLAY);
     while (1) {
         if (rpy_revdb.lock == 0) {
@@ -345,7 +346,9 @@
     *rpy_active_thread_ptr = 0;
     rpy_active_thread = 1;
     rpy_active_thread_ptr = &rpy_active_thread;
-    emit_async_block(ASYNC_THREAD_SWITCH, (uint64_t)pthread_self());
+    pself = (uint64_t)pthread_self();
+    emit_async_block(ASYNC_THREAD_SWITCH, pself);
+    _RPY_REVDB_PRINT("[THRD]", pself);
 }
 
 static void record_stop_point(void)
@@ -362,7 +365,9 @@
     int64_t done;
 
     /* Write an ASYNC_FINALIZER_TRIGGER packet */
+    _RPY_REVDB_LOCK();
     emit_async_block(ASYNC_FINALIZER_TRIGGER, rpy_revdb.stop_point_seen);
+    _RPY_REVDB_UNLOCK();
 
     /* Invoke all Boehm finalizers.  For new-style finalizers, this
        will only cause them to move to the queues, where
@@ -854,6 +859,18 @@
     /* rpy_revdb.buf_limit is not set */
 }
 
+static uint64_t fetch_async_block(void)
+{
+    ssize_t full_packet_size = sizeof(int16_t) + sizeof(int64_t);
+    ssize_t keep = rpy_revdb.buf_readend - rpy_revdb.buf_p;
+    uint64_t result;
+    if (keep < full_packet_size)
+        fetch_more(keep, full_packet_size);
+    memcpy(&result, rpy_revdb.buf_p + sizeof(int16_t), sizeof(int64_t));
+    rpy_revdb.buf_p += full_packet_size;
+    return result;
+}
+
 RPY_EXTERN
 void rpy_reverse_db_fetch(const char *file, int line)
 {
@@ -871,6 +888,7 @@
             exit(1);
         }
 
+     read_next_packet:
         keep = rpy_revdb.buf_readend - rpy_revdb.buf_p;
         assert(keep >= 0);
 
@@ -892,11 +910,7 @@
                                     "ASYNC_FINALIZER_TRIGGER\n");
                     exit(1);
                 }
-                full_packet_size = sizeof(int16_t) + sizeof(int64_t);
-                if (keep < full_packet_size)
-                    fetch_more(keep, full_packet_size);
-                memcpy(&bp, rpy_revdb.buf_p + sizeof(int16_t), sizeof(int64_t));
-                rpy_revdb.buf_p += full_packet_size;
+                bp = fetch_async_block();
                 if (bp <= rpy_revdb.stop_point_seen) {
                     fprintf(stderr, "invalid finalizer break point\n");
                     exit(1);
@@ -908,6 +922,10 @@
                 rpy_revdb.buf_limit = rpy_revdb.buf_p;
                 return;
 
+            case ASYNC_THREAD_SWITCH:
+                fetch_async_block();
+                goto read_next_packet;
+
             default:
                 fprintf(stderr, "bad packet header %d\n", (int)header);
                 exit(1);
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -35,8 +35,8 @@
 #  define _RPY_REVDB_PRINT(mode, _e)                                    \
     if (rpy_rev_fileno >= 0) {                                          \
         fprintf(stderr,                                                 \
-                "%s:%d: %0*llx\n",                                      \
-                __FILE__, __LINE__, 2 * sizeof(_e),                     \
+                "%s %s:%d: %0*llx\n",                                   \
+                mode, __FILE__, __LINE__, 2 * sizeof(_e),               \
                 ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)); \
     }
 #endif
@@ -47,7 +47,7 @@
     if (rpy_rev_fileno >= 0) {                                          \
         seeing_uid(uid);                                                \
         fprintf(stderr,                                                 \
-                "%s:%d: obj %llu\n",                                    \
+                "[nobj] %s:%d: obj %llu\n",                             \
                 __FILE__, __LINE__, (unsigned long long) uid);          \
     }
 #endif
@@ -76,7 +76,7 @@
 #define _RPY_REVDB_EMIT_RECORD_L(decl_e, variable)                      \
         {                                                               \
             decl_e = variable;                                          \
-            _RPY_REVDB_PRINT("write", _e);                              \
+            _RPY_REVDB_PRINT("[ wr ]", _e);                             \
             memcpy(rpy_revdb.buf_p, &_e, sizeof(_e));                   \
             if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit)  \
                 rpy_reverse_db_flush();                                 \
@@ -89,7 +89,7 @@
             char *_end1 = _src + sizeof(_e);                            \
             memcpy(&_e, _src, sizeof(_e));                              \
             rpy_revdb.buf_p = _end1;                                    \
-            _RPY_REVDB_PRINT("read", _e);                               \
+            _RPY_REVDB_PRINT("[read]", _e);                             \
             if (_end1 >= rpy_revdb.buf_limit)                           \
                 rpy_reverse_db_fetch(__FILE__, __LINE__);               \
             variable = _e;                                              \
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -13,6 +13,9 @@
 from rpython.translator.revdb.process import ReplayProcess
 
 
+ASYNC_THREAD_SWITCH = 0xff54 - 2**16
+
+
 class RDB(object):
     def __init__(self, filename, expected_argv):
         with open(filename, 'rb') as f:
@@ -30,6 +33,7 @@
         self.argc = self.read1('i')
         self.argv = self.read1('P')
         self.current_packet_end = self.cur
+        self.main_thread_id = self.switch_thread()
         self.read_check_argv(expected_argv)
 
     def read1(self, mode):
@@ -95,15 +99,22 @@
     def same_stack(self):
         x = self.next('c'); assert x == '\xFC'
 
+    def switch_thread(self, expected=None):
+        th, = self.special_packet(ASYNC_THREAD_SWITCH, 'q')
+        if expected is not None:
+            assert th == expected
+        return th
+
 
 def compile(self, entry_point, backendopt=True,
-            withsmallfuncsets=None, shared=False):
+            withsmallfuncsets=None, shared=False, thread=False):
     t = Translation(entry_point, None, gc="boehm")
     self.t = t
     t.set_backend_extra_options(c_debug_defines=True)
     t.config.translation.reverse_debugger = True
     t.config.translation.lldebug0 = True
     t.config.translation.shared = shared
+    t.config.translation.thread = thread
     if withsmallfuncsets is not None:
         t.config.translation.withsmallfuncsets = withsmallfuncsets
     if not backendopt:

From pypy.commits at gmail.com  Tue Aug  9 16:45:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 13:45:42 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57aa40f6.274fc20a.a9df1.b17b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r776:8ade9f34749c
Date: 2016-08-09 22:45 +0200
http://bitbucket.org/pypy/pypy.org/changeset/8ade9f34749c/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $64800 of $105000 (61.7%)
+   $64836 of $105000 (61.7%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5283 left
+    we have $5314 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $30845 of $80000 (38.6%)
+   $30855 of $80000 (38.6%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $23502 left
+    we have $23511 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Tue Aug  9 19:32:16 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 16:32:16 -0700 (PDT)
Subject: [pypy-commit] pypy py2-mappingproxy: Close branch py2-mappingproxy
Message-ID: <57aa6800.94071c0a.23329.e037@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py2-mappingproxy
Changeset: r86123:812334df02d1
Date: 2016-08-10 00:31 +0100
http://bitbucket.org/pypy/pypy/changeset/812334df02d1/

Log:	Close branch py2-mappingproxy


From pypy.commits at gmail.com  Tue Aug  9 19:32:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 09 Aug 2016 16:32:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in py2-mappingproxy (pull
 request #470)
Message-ID: <57aa6826.94a51c0a.3a1e7.e582@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86124:1900672c0757
Date: 2016-08-10 00:31 +0100
http://bitbucket.org/pypy/pypy/changeset/1900672c0757/

Log:	Merged in py2-mappingproxy (pull request #470)

	Fix the dictproxy type to behave as in CPython. App-level
	cls.__dict__ and C-level cls->tp_dict now return different objects
	with the former being an opaque (at app-level) wrapper around the
	latter.

diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -175,7 +175,7 @@
          "__new__"      : make_new(class_name),
          }
     pycppclass = metacpp(class_name, _drop_cycles(bases), d)
- 
+
     # cache result early so that the class methods can find the class itself
     setattr(scope, final_class_name, pycppclass)
 
@@ -192,13 +192,10 @@
     for dm_name in cppclass.get_datamember_names():
         cppdm = cppclass.get_datamember(dm_name)
 
-        # here, setattr() can not be used, because a data member can shadow one in
-        # its base class, resulting in the __set__() of its base class being called
-        # by setattr(); so, store directly on the dictionary
-        pycppclass.__dict__[dm_name] = cppdm
+        setattr(pycppclass, dm_name, cppdm)
         import cppyy
         if cppyy._is_static(cppdm):     # TODO: make this a method of cppdm
-            metacpp.__dict__[dm_name] = cppdm
+            setattr(metacpp, dm_name, cppdm)
 
     # the call to register will add back-end specific pythonizations and thus
     # needs to run first, so that the generic pythonizations can use them
@@ -413,7 +410,7 @@
         lib = cppyy._load_dictionary(name)
         _loaded_dictionaries[name] = lib
         return lib
-    
+
 def _init_pythonify():
     # cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause cppyy to be loaded
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -282,11 +282,41 @@
                      args->ob_type->tp_dict, "copy");
                  Py_INCREF(method);
                  return method;
+             '''),
+            ("get_type_dict", "METH_O",
              '''
-             )
+                PyObject* value = args->ob_type->tp_dict;
+                if (value == NULL) value = Py_None;
+                Py_INCREF(value);
+                return value;
+             '''),
             ])
         obj = foo.new()
         assert module.read_tp_dict(obj) == foo.fooType.copy
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
+        class A(object):
+            pass
+        obj = A()
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
+        d = module.get_type_dict(1)
+        assert type(d) is dict
+        try:
+            d["_some_attribute"] = 1
+        except TypeError:  # on PyPy, int.__dict__ is really immutable
+            pass
+        else:
+            assert int._some_attribute == 1
+            del d["_some_attribute"]
 
     def test_custom_allocation(self):
         foo = self.import_module("foo")
@@ -355,6 +385,21 @@
 
         api.Py_DecRef(ref)
 
+    def test_type_dict(self, space, api):
+        w_class = space.appexec([], """():
+            class A(object):
+                pass
+            return A
+            """)
+        ref = make_ref(space, w_class)
+
+        py_type = rffi.cast(PyTypeObjectPtr, ref)
+        w_dict = from_ref(space, py_type.c_tp_dict)
+        w_name = space.wrap('a')
+        space.setitem(w_dict, w_name, space.wrap(1))
+        assert space.int_w(space.getattr(w_class, w_name)) == 1
+        space.delitem(w_dict, w_name)
+
     def test_multiple_inheritance(self, space, api):
         w_class = space.appexec([], """():
             class A(object):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -272,12 +272,12 @@
         if len(slot_names) == 1:
             if not getattr(pto, slot_names[0]):
                 setattr(pto, slot_names[0], slot_func_helper)
-        elif (w_type.getname(space) in ('list', 'tuple') and 
+        elif (w_type.getname(space) in ('list', 'tuple') and
               slot_names[0] == 'c_tp_as_number'):
             # XXX hack - hwo can we generalize this? The problem is method
             # names like __mul__ map to more than one slot, and we have no
             # convenient way to indicate which slots CPython have filled
-            # 
+            #
             # We need at least this special case since Numpy checks that
             # (list, tuple) do __not__ fill tp_as_number
             pass
@@ -860,8 +860,8 @@
 
     if w_obj.is_cpytype():
         Py_DecRef(space, pto.c_tp_dict)
-        w_dict = w_obj.getdict(space)
-        pto.c_tp_dict = make_ref(space, w_dict)
+    w_dict = w_obj.getdict(space)
+    pto.c_tp_dict = make_ref(space, w_dict)
 
 @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL)
 def PyType_IsSubtype(space, a, b):
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/classdict.py
copy from pypy/objspace/std/dictproxyobject.py
copy to pypy/objspace/std/classdict.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/classdict.py
@@ -7,7 +7,7 @@
 from pypy.objspace.std.typeobject import unwrap_cell
 
 
-class DictProxyStrategy(DictStrategy):
+class ClassDictStrategy(DictStrategy):
     erase, unerase = rerased.new_erasing_pair("dictproxy")
     erase = staticmethod(erase)
     unerase = staticmethod(unerase)
@@ -85,7 +85,8 @@
         return space.newlist_bytes(self.unerase(w_dict.dstorage).dict_w.keys())
 
     def values(self, w_dict):
-        return [unwrap_cell(self.space, w_value) for w_value in self.unerase(w_dict.dstorage).dict_w.itervalues()]
+        return [unwrap_cell(self.space, w_value) for w_value in
+                self.unerase(w_dict.dstorage).dict_w.itervalues()]
 
     def items(self, w_dict):
         space = self.space
@@ -103,13 +104,17 @@
 
     def getiterkeys(self, w_dict):
         return self.unerase(w_dict.dstorage).dict_w.iterkeys()
+
     def getitervalues(self, w_dict):
         return self.unerase(w_dict.dstorage).dict_w.itervalues()
+
     def getiteritems_with_hash(self, w_dict):
         return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w)
+
     def wrapkey(space, key):
         return space.wrap(key)
+
     def wrapvalue(space, value):
         return unwrap_cell(space, value)
 
-create_iterator_classes(DictProxyStrategy)
+create_iterator_classes(ClassDictStrategy)
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -1,115 +1,101 @@
-from rpython.rlib import rerased
-from rpython.rlib.objectmodel import iteritems_with_hash
+"""
+Read-only proxy for mappings.
 
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.objspace.std.dictmultiobject import (
-    DictStrategy, create_iterator_classes)
-from pypy.objspace.std.typeobject import unwrap_cell
+Its main use is as the return type of cls.__dict__.
+"""
 
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
+from pypy.interpreter.typedef import TypeDef, interp2app
 
-class DictProxyStrategy(DictStrategy):
-    erase, unerase = rerased.new_erasing_pair("dictproxy")
-    erase = staticmethod(erase)
-    unerase = staticmethod(unerase)
+class W_DictProxyObject(W_Root):
+    "Read-only proxy for mappings."
 
-    def __init__(self, space):
-        DictStrategy.__init__(self, space)
+    def __init__(self, w_mapping):
+        self.w_mapping = w_mapping
 
-    def getitem(self, w_dict, w_key):
-        space = self.space
-        w_lookup_type = space.type(w_key)
-        if (space.is_w(w_lookup_type, space.w_str) or  # Most common path first
-            space.abstract_issubclass_w(w_lookup_type, space.w_str)):
-            return self.getitem_str(w_dict, space.str_w(w_key))
-        elif space.abstract_issubclass_w(w_lookup_type, space.w_unicode):
-            try:
-                w_key = space.str(w_key)
-            except OperationError as e:
-                if not e.match(space, space.w_UnicodeEncodeError):
-                    raise
-                # non-ascii unicode is never equal to a byte string
-                return None
-            return self.getitem_str(w_dict, space.str_w(w_key))
-        else:
-            return None
+    @staticmethod
+    def descr_new(space, w_type, w_mapping):
+        raise oefmt(space.w_TypeError, "Cannot create 'dictproxy' instances")
 
-    def getitem_str(self, w_dict, key):
-        return self.unerase(w_dict.dstorage).getdictvalue(self.space, key)
+    def descr_init(self, space, __args__):
+        pass
 
-    def setitem(self, w_dict, w_key, w_value):
-        space = self.space
-        if space.is_w(space.type(w_key), space.w_str):
-            self.setitem_str(w_dict, self.space.str_w(w_key), w_value)
-        else:
-            raise oefmt(space.w_TypeError,
-                        "cannot add non-string keys to dict of a type")
+    def descr_len(self, space):
+        return space.len(self.w_mapping)
 
-    def setitem_str(self, w_dict, key, w_value):
-        w_type = self.unerase(w_dict.dstorage)
-        try:
-            w_type.setdictvalue(self.space, key, w_value)
-        except OperationError as e:
-            if not e.match(self.space, self.space.w_TypeError):
-                raise
-            if not w_type.is_cpytype():
-                raise
-            # Allow cpyext to write to type->tp_dict even in the case
-            # of a builtin type.
-            # Like CPython, we assume that this is only done early
-            # after the type is created, and we don't invalidate any
-            # cache.  User code shoud call PyType_Modified().
-            w_type.dict_w[key] = w_value
+    def descr_getitem(self, space, w_key):
+        return space.getitem(self.w_mapping, w_key)
 
-    def setdefault(self, w_dict, w_key, w_default):
-        w_result = self.getitem(w_dict, w_key)
-        if w_result is not None:
-            return w_result
-        self.setitem(w_dict, w_key, w_default)
-        return w_default
+    def descr_contains(self, space, w_key):
+        return space.contains(self.w_mapping, w_key)
 
-    def delitem(self, w_dict, w_key):
-        space = self.space
-        w_key_type = space.type(w_key)
-        if space.is_w(w_key_type, space.w_str):
-            key = self.space.str_w(w_key)
-            if not self.unerase(w_dict.dstorage).deldictvalue(space, key):
-                raise KeyError
-        else:
-            raise KeyError
+    def descr_iter(self, space):
+        return space.iter(self.w_mapping)
 
-    def length(self, w_dict):
-        return len(self.unerase(w_dict.dstorage).dict_w)
+    def descr_str(self, space):
+        return space.str(self.w_mapping)
 
-    def w_keys(self, w_dict):
-        space = self.space
-        return space.newlist_bytes(self.unerase(w_dict.dstorage).dict_w.keys())
+    def descr_repr(self, space):
+        return space.wrap("dict_proxy(%s)" %
+                                (space.str_w(space.repr(self.w_mapping)),))
 
-    def values(self, w_dict):
-        return [unwrap_cell(self.space, w_value) for w_value in self.unerase(w_dict.dstorage).dict_w.itervalues()]
+    @unwrap_spec(w_default=WrappedDefault(None))
+    def get_w(self, space, w_key, w_default):
+        return space.call_method(self.w_mapping, "get", w_key, w_default)
 
-    def items(self, w_dict):
-        space = self.space
-        return [space.newtuple([space.wrap(key), unwrap_cell(self.space, w_value)])
-                    for (key, w_value) in self.unerase(w_dict.dstorage).dict_w.iteritems()]
+    def keys_w(self, space):
+        return space.call_method(self.w_mapping, "keys")
 
-    def clear(self, w_dict):
-        space = self.space
-        w_type = self.unerase(w_dict.dstorage)
-        if not w_type.is_heaptype():
-            raise oefmt(space.w_TypeError,
-                        "can't clear dictionary of type '%N'", w_type)
-        w_type.dict_w.clear()
-        w_type.mutated(None)
+    def descr_iterkeys(self, space):
+        return space.call_method(self.w_mapping, "iterkeys")
 
-    def getiterkeys(self, w_dict):
-        return self.unerase(w_dict.dstorage).dict_w.iterkeys()
-    def getitervalues(self, w_dict):
-        return self.unerase(w_dict.dstorage).dict_w.itervalues()
-    def getiteritems_with_hash(self, w_dict):
-        return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w)
-    def wrapkey(space, key):
-        return space.wrap(key)
-    def wrapvalue(space, value):
-        return unwrap_cell(space, value)
+    def values_w(self, space):
+        return space.call_method(self.w_mapping, "values")
 
-create_iterator_classes(DictProxyStrategy)
+    def descr_itervalues(self, space):
+        return space.call_method(self.w_mapping, "itervalues")
+
+    def items_w(self, space):
+        return space.call_method(self.w_mapping, "items")
+
+    def descr_iteritems(self, space):
+        return space.call_method(self.w_mapping, "iteritems")
+
+    def copy_w(self, space):
+        return space.call_method(self.w_mapping, "copy")
+
+cmp_methods = {}
+def make_cmp_method(op):
+    def descr_op(self, space, w_other):
+        return getattr(space, op)(self.w_mapping, w_other)
+    descr_name = 'descr_' + op
+    descr_op.__name__ = descr_name
+    setattr(W_DictProxyObject, descr_name, descr_op)
+    cmp_methods['__%s__' % op] = interp2app(getattr(W_DictProxyObject, descr_name))
+
+for op in ['eq', 'ne', 'gt', 'ge', 'lt', 'le']:
+    make_cmp_method(op)
+
+
+W_DictProxyObject.typedef = TypeDef(
+    'dictproxy',
+    __new__=interp2app(W_DictProxyObject.descr_new),
+    __init__=interp2app(W_DictProxyObject.descr_init),
+    __len__=interp2app(W_DictProxyObject.descr_len),
+    __getitem__=interp2app(W_DictProxyObject.descr_getitem),
+    __contains__=interp2app(W_DictProxyObject.descr_contains),
+    __iter__=interp2app(W_DictProxyObject.descr_iter),
+    __str__=interp2app(W_DictProxyObject.descr_str),
+    __repr__=interp2app(W_DictProxyObject.descr_repr),
+    get=interp2app(W_DictProxyObject.get_w),
+    keys=interp2app(W_DictProxyObject.keys_w),
+    iterkeys=interp2app(W_DictProxyObject.descr_iterkeys),
+    values=interp2app(W_DictProxyObject.values_w),
+    itervalues=interp2app(W_DictProxyObject.descr_itervalues),
+    items=interp2app(W_DictProxyObject.items_w),
+    iteritems=interp2app(W_DictProxyObject.descr_iteritems),
+    copy=interp2app(W_DictProxyObject.copy_w),
+    **cmp_methods
+)
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -9,37 +9,19 @@
         assert 'a' in NotEmpty.__dict__
         assert 'a' in NotEmpty.__dict__.keys()
         assert 'b' not in NotEmpty.__dict__
-        NotEmpty.__dict__['b'] = 4
-        assert NotEmpty.b == 4
-        del NotEmpty.__dict__['b']
         assert NotEmpty.__dict__.get("b") is None
+        raises(TypeError, "NotEmpty.__dict__['b'] = 4")
         raises(TypeError, 'NotEmpty.__dict__[15] = "y"')
-        raises(KeyError, 'del NotEmpty.__dict__[15]')
+        raises(TypeError, 'del NotEmpty.__dict__[15]')
 
-        assert NotEmpty.__dict__.setdefault("string", 1) == 1
-        assert NotEmpty.__dict__.setdefault("string", 2) == 1
-        assert NotEmpty.string == 1
-        raises(TypeError, 'NotEmpty.__dict__.setdefault(15, 1)')
-
-    def test_dictproxy_popitem(self):
-        class A(object):
-            a = 42
-        seen = 0
-        try:
-            while True:
-                key, value = A.__dict__.popitem()
-                if key == 'a':
-                    assert value == 42
-                    seen += 1
-        except KeyError:
-            pass
-        assert seen == 1
+        raises(AttributeError, 'NotEmpty.__dict__.setdefault')
 
     def test_dictproxy_getitem(self):
         class NotEmpty(object):
             a = 1
         assert 'a' in NotEmpty.__dict__
-        class substr(str): pass
+        class substr(str):
+            pass
         assert substr('a') in NotEmpty.__dict__
         assert u'a' in NotEmpty.__dict__
         assert NotEmpty.__dict__[u'a'] == 1
@@ -62,15 +44,40 @@
         class a(object):
             pass
         s1 = repr(a.__dict__)
+        assert s1.startswith('dict_proxy({') and s1.endswith('})')
         s2 = str(a.__dict__)
-        assert s1 == s2
-        assert s1.startswith('{') and s1.endswith('}')
+        assert s1 == 'dict_proxy(%s)' % s2
 
     def test_immutable_dict_on_builtin_type(self):
         raises(TypeError, "int.__dict__['a'] = 1")
-        raises(TypeError, int.__dict__.popitem)
-        raises(TypeError, int.__dict__.clear)
+        raises((AttributeError, TypeError), "int.__dict__.popitem()")
+        raises((AttributeError, TypeError), "int.__dict__.clear()")
+
+    def test_dictproxy(self):
+        dictproxy = type(int.__dict__)
+        assert dictproxy is not dict
+        assert dictproxy.__name__ == 'dictproxy'
+        raises(TypeError, dictproxy)
+
+        mapping = {'a': 1}
+        raises(TypeError, dictproxy, mapping)
+
+        class A(object):
+            a = 1
+
+        proxy = A.__dict__
+        mapping = dict(proxy)
+        assert proxy['a'] == 1
+        assert 'a' in proxy
+        assert 'z' not in proxy
+        assert repr(proxy) == 'dict_proxy(%r)' % mapping
+        assert proxy.keys() == mapping.keys()
+        assert list(proxy.iterkeys()) == proxy.keys()
+        assert list(proxy.itervalues()) == proxy.values()
+        assert list(proxy.iteritems()) == proxy.items()
+        raises(TypeError, "proxy['a'] = 4")
+        raises(TypeError, "del proxy['a']")
+        raises(AttributeError, "proxy.clear()")
 
 class AppTestUserObjectMethodCache(AppTestUserObject):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
-
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -968,7 +968,6 @@
         raises(TypeError, setattr, list, 'foobar', 42)
         raises(TypeError, delattr, dict, 'keys')
         raises(TypeError, 'int.__dict__["a"] = 1')
-        raises(TypeError, 'int.__dict__.clear()')
 
     def test_nontype_in_mro(self):
         class OldStyle:
@@ -1026,10 +1025,9 @@
             pass
 
         a = A()
+        d = A.__dict__
         A.x = 1
-        assert A.__dict__["x"] == 1
-        A.__dict__['x'] = 5
-        assert A.x == 5
+        assert d["x"] == 1
 
     def test_we_already_got_one_1(self):
         # Issue #2079: highly obscure: CPython complains if we say
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -3,8 +3,8 @@
 from pypy.interpreter.baseobjspace import W_Root, SpaceCache
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import Function, StaticMethod
-from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\
-     descr_get_dict, dict_descr, Member, TypeDef
+from pypy.interpreter.typedef import (
+    weakref_descr, GetSetProperty, dict_descr, Member, TypeDef)
 from pypy.interpreter.astcompiler.misc import mangle
 from pypy.module.__builtin__ import abstractinst
 
@@ -346,7 +346,7 @@
     def deldictvalue(self, space, key):
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        if not self.is_heaptype():
+        if not (self.is_heaptype() or self.is_cpytype()):
             raise oefmt(space.w_TypeError,
                         "can't delete attributes on type object '%N'", self)
         try:
@@ -485,12 +485,12 @@
                 self.getdictvalue(self.space, attr)
             del self.lazyloaders
 
-    def getdict(self, space): # returning a dict-proxy!
-        from pypy.objspace.std.dictproxyobject import DictProxyStrategy
+    def getdict(self, space):
+        from pypy.objspace.std.classdict import ClassDictStrategy
         from pypy.objspace.std.dictmultiobject import W_DictObject
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        strategy = space.fromcache(DictProxyStrategy)
+        strategy = space.fromcache(ClassDictStrategy)
         storage = strategy.erase(self)
         return W_DictObject(space, strategy, storage)
 
@@ -910,13 +910,21 @@
     return space.newbool(
         abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj))
 
+def type_get_dict(space, w_cls):
+    w_cls = _check(space, w_cls)
+    from pypy.objspace.std.dictproxyobject import W_DictProxyObject
+    w_dict = w_cls.getdict(space)
+    if w_dict is None:
+        return space.w_None
+    return W_DictProxyObject(w_dict)
+
 W_TypeObject.typedef = TypeDef("type",
     __new__ = gateway.interp2app(descr__new__),
     __name__ = GetSetProperty(descr_get__name__, descr_set__name__),
     __bases__ = GetSetProperty(descr_get__bases__, descr_set__bases__),
     __base__ = GetSetProperty(descr__base),
     __mro__ = GetSetProperty(descr_get__mro__),
-    __dict__ = GetSetProperty(descr_get_dict),
+    __dict__=GetSetProperty(type_get_dict),
     __doc__ = GetSetProperty(descr__doc),
     mro = gateway.interp2app(descr_mro),
     __flags__ = GetSetProperty(descr__flags),

From pypy.commits at gmail.com  Wed Aug 10 00:02:09 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 09 Aug 2016 21:02:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: test, fix PySequence_Fast getslice;
 remove outdated document
Message-ID: <57aaa741.c2f3c20a.a80d5.1951@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86125:72d14a4de609
Date: 2016-08-10 07:01 +0300
http://bitbucket.org/pypy/pypy/changeset/72d14a4de609/

Log:	test, fix PySequence_Fast getslice; remove outdated document

diff --git a/pypy/module/cpyext/c-api.txt b/pypy/module/cpyext/c-api.txt
deleted file mode 100644
--- a/pypy/module/cpyext/c-api.txt
+++ /dev/null
@@ -1,71 +0,0 @@
-Reference Count
-===============
-
-XXX
-
-Borrowed References
-===================
-
-XXX
-
-PyStringObject support
-======================
-
-The problem
------------
-
-PyString_AsString() returns a (non-movable) pointer to the underlying
-buffer, whereas pypy strings are movable.  C code may temporarily
-store this address and use it, as long as it owns a reference to the
-PyObject.  There is no "release" function to specify that the pointer
-is not needed any more.
-
-Note that the pointer may be used to fill the initial value of
-string. This is valid only when the string was just allocated, and is
-not used elsewhere.
-
-Proposed solution
------------------
-
-Our emulation of the PyStringObject contains an additional member: a
-pointer to a char buffer; it may be NULL.
-
-- A string allocated by pypy will be converted into a PyStringObject
-  with a NULL buffer.  When PyString_AsString() is called, memory is
-  allocated (with flavor='raw') and content is copied.
-
-- A string allocated with PyString_FromStringAndSize(NULL, size) will
-  allocate a buffer with the specified size, but the reference won't
-  be stored in the global map py_objects_r2w; there won't be a
-  corresponding object in pypy.  When from_ref() or Py_INCREF() is
-  called, the pypy string is created, and added in py_objects_r2w.
-  The buffer is then supposed to be immutable.
-
-- _PyString_Resize works only on not-yet-pypy'd strings, and returns a
-  similar object.
-
-- PyString_Size don't need to force the object. (in this case, another
-  "size" member is needed)
-
-- There could be an (expensive!) check in from_ref() that the buffer
-  still corresponds to the pypy gc-managed string.
-
-PySequence_Fast support
-======================
-There are five functions for fast sequence access offered by the CPython API:
-
-PyObject* PySequence_Fast(PyObject *o, const char *m)
-
-PyObject* PySequence_Fast_GET_ITEM(	PyObject *o, int i)
-
-PyObject** PySequence_Fast_ITEMS(	PyObject *o)
-
-PyObject* PySequence_ITEM(	PyObject *o, int i)
-
-int PySequence_Fast_GET_SIZE(	PyObject *o)
-
-PyPy supports four of these, but does not support PySequence_Fast_ITEMS.
-(Various ways to support PySequence_Fast_ITEMS were considered. They all had
-two things in common: they would have taken a lot of work, and they would have
-resulted in incomplete semantics or in poor performance. We decided that a slow
-implementation of PySequence_Fast_ITEMS was not very useful.)
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -10,7 +10,7 @@
 from pypy.objspace.std import tupleobject
 
 from pypy.module.cpyext.tupleobject import PyTuple_Check, PyTuple_SetItem
-from pypy.module.cpyext.object import Py_IncRef, Py_DecRef
+from pypy.module.cpyext.pyobject import decref
 
 from pypy.module.cpyext.dictobject import PyDict_Check
 
@@ -252,7 +252,7 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        Py_DecRef(w_list.space, storage._elems[index])
+        decref(w_list.space, storage._elems[index])
         storage._elems[index] = make_ref(w_list.space, w_obj)
 
     def length(self, w_list):
@@ -264,9 +264,8 @@
         return storage._elems
 
     def getslice(self, w_list, start, stop, step, length):
-        #storage = self.unerase(w_list.lstorage)
-        raise oefmt(w_list.space.w_NotImplementedError,
-                    "settting a slice of a PySequence_Fast is not supported")
+        w_list.switch_to_object_strategy()
+        return w_list.strategy.getslice(w_list, start, stop, step, length)
 
     def getitems(self, w_list):
         # called when switching list strategy, so convert storage
@@ -389,5 +388,5 @@
 
     def __del__(self):
         for i in range(self._length):
-            Py_DecRef(self.space, self._elems[i])
+            decref(self.space, self._elems[i])
         lltype.free(self._elems, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -78,6 +78,17 @@
         assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
         assert space.eq_w(w_t, space.wrap([1, 3, 5]))
 
+    def test_get_slice_fast(self, space, api):
+        w_t = space.wrap([1, 2, 3, 4, 5])
+        api.PySequence_Fast(w_t, "foo") # converts
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4]
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4]
+
+        assert api.PySequence_DelSlice(w_t, 1, 4) == 0
+        assert space.eq_w(w_t, space.wrap([1, 5]))
+        assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
+        assert space.eq_w(w_t, space.wrap([1, 3, 5]))
+
     def test_iter(self, space, api):
         w_t = space.wrap((1, 2))
         w_iter = api.PySeqIter_New(w_t)
@@ -226,18 +237,33 @@
         assert space.int_w(space.len(w_l)) == 10
 
 
-class XAppTestSequenceObject(AppTestCpythonExtensionBase):
-    def test_sequenceobject(self):
+class AppTestSequenceObject(AppTestCpythonExtensionBase):
+    def test_fast(self):
         module = self.import_extension('foo', [
             ("test_fast_sequence", "METH_VARARGS",
              """
-                PyObject * o = PyTuple_GetItem(args, 0);
+                int size, i;
+                PyTypeObject * common_type;
+                PyObject *foo, **objects;
+                PyObject * seq = PyTuple_GetItem(args, 0);
                 /* XXX assert it is a tuple */
-                PyObject *foo = PySequence_Fast(o, "some string");
-                PyObject ** res = PySequence_Fast_ITEMS(foo);
-                /* XXX do some kind of test on res */
-                /* XXX now what? who manages res's refcount? */
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "some string");
+                objects = PySequence_Fast_ITEMS(foo);
+                size = PySequence_Fast_GET_SIZE(seq);
+                common_type = size > 0 ? Py_TYPE(objects[0]) : NULL;
+                for (i = 1; i < size; ++i) {
+                    if (Py_TYPE(objects[i]) != common_type) {
+                        common_type = NULL;
+                        break;
+                    }
+                }
+                Py_DECREF(foo);
+                Py_DECREF(common_type);
                 return PyBool_FromLong(1);
              """)])
-        assert module.test_fast_sequence([1, 2, 3, 4])
+        s = [1, 2, 3, 4]
+        assert module.test_fast_sequence(s[0:-1])
+        assert module.test_fast_sequence(s[::-1])
 

From pypy.commits at gmail.com  Wed Aug 10 02:35:25 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 09 Aug 2016 23:35:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async-translate: close branch
Message-ID: <57aacb2d.271ac20a.e6d8c.317a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async-translate
Changeset: r86126:2569ef7fa3cc
Date: 2016-08-10 08:34 +0200
http://bitbucket.org/pypy/pypy/changeset/2569ef7fa3cc/

Log:	close branch


From pypy.commits at gmail.com  Wed Aug 10 02:43:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 Aug 2016 23:43:46 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fix locking logic
Message-ID: <57aacd22.08d11c0a.7d6c5.4372@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86127:a0e97b020056
Date: 2016-08-10 08:43 +0200
http://bitbucket.org/pypy/pypy/changeset/a0e97b020056/

Log:	fix locking logic

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -331,11 +331,11 @@
 }
 
 RPY_EXTERN
-void rpy_reverse_db_lock_acquire(void)
+void rpy_reverse_db_lock_acquire(bool_t lock_contention)
 {
     uint64_t pself;
     assert(!RPY_RDB_REPLAY);
-    while (1) {
+    while (lock_contention) {
         if (rpy_revdb.lock == 0) {
             if (pypy_lock_test_and_set(&rpy_revdb.lock, 1) == 0)
                 break;   /* done */
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -65,10 +65,11 @@
    single-threaded during replaying: the lock is only useful during
    recording. */
 #define _RPY_REVDB_LOCK()                                               \
-    if (!rpy_active_thread ||                                           \
-            pypy_lock_test_and_set(&rpy_revdb.lock, 1) != 0)            \
-        rpy_reverse_db_lock_acquire();
-
+    {                                                                   \
+        bool_t _lock_contention = pypy_lock_test_and_set(&rpy_revdb.lock, 1); \
+        if (_lock_contention || !rpy_active_thread)                     \
+            rpy_reverse_db_lock_acquire(_lock_contention);              \
+    }
 #define _RPY_REVDB_UNLOCK()                                             \
     pypy_lock_release(&rpy_revdb.lock)
 
@@ -221,6 +222,6 @@
 RPY_EXTERN void rpy_reverse_db_call_destructor(void *obj);
 RPY_EXTERN void rpy_reverse_db_invoke_callback(unsigned char);
 RPY_EXTERN void rpy_reverse_db_callback_loc(int);
-RPY_EXTERN void rpy_reverse_db_lock_acquire(void);
+RPY_EXTERN void rpy_reverse_db_lock_acquire(bool_t lock_contention);
 
 /* ------------------------------------------------------------ */

From pypy.commits at gmail.com  Wed Aug 10 03:26:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 10 Aug 2016 00:26:15 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fixes
Message-ID: <57aad717.d8011c0a.c929d.5215@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86128:33ad82b0c795
Date: 2016-08-10 09:25 +0200
http://bitbucket.org/pypy/pypy/changeset/33ad82b0c795/

Log:	fixes

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -203,7 +203,8 @@
     # use generate_source(defines=DEBUG_DEFINES) to force the #definition
     # of the macros that enable debugging assertions
     DEBUG_DEFINES = {'RPY_ASSERT': 1,
-                     'RPY_LL_ASSERT': 1}
+                     'RPY_LL_ASSERT': 1,
+                     'RPY_REVDB_PRINT_ALL': 1}
 
     def generate_source(self, db=None, defines={}, exe_name=None):
         assert self.c_source_filename is None
diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py
--- a/rpython/translator/revdb/gencsupp.py
+++ b/rpython/translator/revdb/gencsupp.py
@@ -83,9 +83,18 @@
         return call_code   # a hack for ll_call_destructor() to mean
                            # that the calls should really be done
     #
-    # hack: we don't need the flag for at least these two common functions
-    if call_code in ('RPyGilRelease();', 'RPyGilAcquire();'):
+    # hack: we don't need the flag for at least this common function
+    if call_code == 'RPyGilRelease();':
         return 'RPY_REVDB_CALL_GILCTRL(%s);' % (call_code,)
+    if call_code == 'RPyGilAcquire();':
+        # Could also work with a regular RPY_REVDB_CALL_VOID, but we
+        # use a different byte (0xFD instead of 0xFC) to detect more
+        # sync misses.  In a single-threaded environment this 0xFD
+        # byte is not needed at all, but in a multi-threaded
+        # environment it ensures that during replaying, we don't go
+        # past the RPyGilAcquire() in case a different thread must run
+        # next.
+        return 'RPY_REVDB_CALL_GIL(%s);' % (call_code,)
     #
     tp = funcgen.lltypename(v_result)
     if tp == 'void @':
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1526,6 +1526,13 @@
     } while (e != 0xFC);
 }
 
+RPY_EXTERN
+void rpy_reverse_db_bad_acquire_gil(void)
+{
+    fprintf(stderr, "out of sync: unexpected byte in log (at acquire_gil)\n");
+    exit(1);
+}
+
 
 /* ------------------------------------------------------------ */
 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -31,7 +31,8 @@
 RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]);
 RPY_EXTERN void rpy_reverse_db_teardown(void);
 
-#if 0    /* enable to print locations to stderr of all the EMITs */
+/* enable to print locations to stderr of all the EMITs */
+#ifdef RPY_REVDB_PRINT_ALL
 #  define _RPY_REVDB_PRINT(mode, _e)                                    \
     if (rpy_rev_fileno >= 0) {                                          \
         fprintf(stderr,                                                 \
@@ -41,7 +42,8 @@
     }
 #endif
 
-#if 0    /* enable to print all mallocs to stderr */
+/* enable to print all mallocs to stderr */
+#ifdef RPY_REVDB_PRINT_ALL
 RPY_EXTERN void seeing_uid(uint64_t uid);
 #  define _RPY_REVDB_PRUID()                                            \
     if (rpy_rev_fileno >= 0) {                                          \
@@ -140,6 +142,20 @@
             rpy_reverse_db_invoke_callback(_re);                        \
     }
 
+#define RPY_REVDB_CALL_GIL(call_code)                                   \
+    if (!RPY_RDB_REPLAY) {                                              \
+        call_code                                                       \
+        _RPY_REVDB_LOCK();                                              \
+        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, 0xFD)                \
+        _RPY_REVDB_UNLOCK();                                            \
+    }                                                                   \
+    else {                                                              \
+        unsigned char _re;                                              \
+        _RPY_REVDB_EMIT_REPLAY(unsigned char _e, _re)                   \
+        if (_re != 0xFD)                                                \
+            rpy_reverse_db_bad_acquire_gil();                           \
+    }
+
 #define RPY_REVDB_CALL_GILCTRL(call_code)                               \
     if (!RPY_RDB_REPLAY) {                                              \
         call_code                                                       \
@@ -223,5 +239,6 @@
 RPY_EXTERN void rpy_reverse_db_invoke_callback(unsigned char);
 RPY_EXTERN void rpy_reverse_db_callback_loc(int);
 RPY_EXTERN void rpy_reverse_db_lock_acquire(bool_t lock_contention);
+RPY_EXTERN void rpy_reverse_db_bad_acquire_gil(void);
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -95,10 +95,14 @@
         x = self.next(); assert x == len(expected_string)
         self.same_stack()   # errno
         x = self.next('i'); assert x == 0      # errno
+        self.gil_acquire()
 
     def same_stack(self):
         x = self.next('c'); assert x == '\xFC'
 
+    def gil_acquire(self):
+        x = self.next('c'); assert x == '\xFD'
+
     def switch_thread(self, expected=None):
         th, = self.special_packet(ASYNC_THREAD_SWITCH, 'q')
         if expected is not None:
diff --git a/rpython/translator/revdb/test/test_callback.py b/rpython/translator/revdb/test/test_callback.py
--- a/rpython/translator/revdb/test/test_callback.py
+++ b/rpython/translator/revdb/test/test_callback.py
@@ -65,13 +65,17 @@
         rdb = self.fetch_rdb([self.exename, 'Xx'])
         rdb.same_stack()                        # callmesimple()
         x = rdb.next('i'); assert x == 55555
+        rdb.gil_acquire()
         rdb.write_call('55555\n')
         b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
         x = rdb.next('i'); assert x == 40       # arg n
+        rdb.gil_acquire()
         x = rdb.next('!h'); assert x == b       # -> callback
         x = rdb.next('i'); assert x == 3        # arg n
+        rdb.gil_acquire()
         rdb.same_stack()                        # <- return in main thread
         x = rdb.next('i'); assert x == 4000 * 300   # return from callme()
+        rdb.gil_acquire()
         rdb.write_call('%s\n' % (4000 * 300,))
         x = rdb.next('q'); assert x == 0      # number of stop points
         assert rdb.done()
@@ -83,12 +87,15 @@
         rdb = self.fetch_rdb([self.exename, 'Xx'])
         b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
         x = rdb.next('i'); assert x == 40       # arg n
+        rdb.gil_acquire()
         rdb.write_call('40\n')
         x = rdb.next('!h'); assert x == b       # -> callback again
         x = rdb.next('i'); assert x == 3        # arg n
+        rdb.gil_acquire()
         rdb.write_call('3\n')
         rdb.same_stack()                        # -> return in main thread
         x = rdb.next('i'); assert x == 120      # <- return from callme()
+        rdb.gil_acquire()
         rdb.write_call('120\n')
         x = rdb.next('q'); assert x == 2        # number of stop points
         assert rdb.done()

From pypy.commits at gmail.com  Wed Aug 10 03:34:18 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 10 Aug 2016 00:34:18 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: pyconza 2016 abstract for general
 pypy talk
Message-ID: <57aad8fa.81cb1c0a.d1959.5383@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5665:66529e75e820
Date: 2016-08-10 09:34 +0200
http://bitbucket.org/pypy/extradoc/changeset/66529e75e820/

Log:	pyconza 2016 abstract for general pypy talk

diff --git a/blog/draft/new-jit-log.rst b/blog/draft/new-jit-log.rst
--- a/blog/draft/new-jit-log.rst
+++ b/blog/draft/new-jit-log.rst
@@ -66,7 +66,7 @@
 Speed issues
 ------------
 
-VMProf is a great tool to find out hot spots that consume a lot of time in your program. As soon as you have identified code that runs slowly, you can switch to jitlog and maybe pinpoint certain aspects that do not behave as expected. You will find an overview, and are able to browse the generated code. If you cannot make sense of all that, you can just share the link with us and we can have a look too.
+VMProf is a great tool to find hot spots that consume a lot of time in your program. As soon as you have identified code that runs slowly, you can switch to jitlog and maybe pinpoint certain aspects that do not behave as expected. You will find an overview, and are able to browse the generated code. If you cannot make sense of all that, you can just share the link with us and we can have a look too.
 
 Future direction
 ----------------
diff --git a/talk/pyconza2016/pypy-abstract.txt b/talk/pyconza2016/pypy-abstract.txt
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/pypy-abstract.txt
@@ -0,0 +1,24 @@
+Optimizing Python programs, PyPy to rescue
+======================================
+
+In this talk I want to show how you can use PyPy for your benefit.
+It will kick off with a short introduction covering PyPy and its just in time
+compiler. PyPy is the most advanced Python interpreter around (besides CPython)
+and while it should generally just speed up your programs there is a wide range
+of performance that you can get out of PyPy.
+
+The first part, will cover considerations why one should write Python programs,
+and only spend fractions of the development time to optimize your program.
+The second part of this session will show and give you the knowledge and
+tools to inspect and change your program to improve it. We will cover two tools in detail:
+CFFI & VMProf.
+
+Our advanced library CFFI (C Foreign Function Interface) can easily replace
+CPython extension code. VMProf is a platform to inspect you program while it is running,
+imposing very little overhead.
+
+Throughout the talk real world examples will motivate why PyPy is a viable option
+to optimize you Python programs and present the examples' value to their developers.
+
+As a result of this talk, an audience member should be equipped with
+tools that helps him to understand and optimize programs. 

From pypy.commits at gmail.com  Wed Aug 10 05:39:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 10 Aug 2016 02:39:20 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Unclear points
Message-ID: <57aaf648.56421c0a.6a424.86fb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5666:d93f9aca2b36
Date: 2016-08-10 11:39 +0200
http://bitbucket.org/pypy/extradoc/changeset/d93f9aca2b36/

Log:	Unclear points

diff --git a/talk/pyconza2016/pypy-abstract.txt b/talk/pyconza2016/pypy-abstract.txt
--- a/talk/pyconza2016/pypy-abstract.txt
+++ b/talk/pyconza2016/pypy-abstract.txt
@@ -1,14 +1,31 @@
-Optimizing Python programs, PyPy to rescue
-======================================
+Optimizing Python programs, PyPy to the rescue
+==============================================
 
 In this talk I want to show how you can use PyPy for your benefit.
 It will kick off with a short introduction covering PyPy and its just in time
 compiler. PyPy is the most advanced Python interpreter around (besides CPython)
+XXX
+XXX you seem to say "CPython is more advanced than PyPy" above,
+XXX which doesn't make sense in this sentence because you say
+XXX below that PyPy is much faster than CPython
+XXX
 and while it should generally just speed up your programs there is a wide range
 of performance that you can get out of PyPy.
 
 The first part, will cover considerations why one should write Python programs,
 and only spend fractions of the development time to optimize your program.
+XXX
+XXX you mean, you want to explain that developers should write in Python
+XXX and spend only a small part of their time optimizing the program?
+XXX or something else?  if I'm right then you should add below something 
+XXX like "The second part of this session will be about this small part
+XXX of time: in cases where you need it, then I'll show tools that..."
+XXX But I'm not sure that's what you mean because CFFI is not really
+XXX about that: I'm trying to push it as a general solution also for
+XXX CPython, without focusing too much on performance.  Maybe we should
+XXX have this talk be really about PyPy, and then for the other talk
+XXX I should have both CFFI and RevDB?
+XXX
 The second part of this session will show and give you the knowledge and
 tools to inspect and change your program to improve it. We will cover two tools in detail:
 CFFI & VMProf.

From pypy.commits at gmail.com  Wed Aug 10 06:25:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 10 Aug 2016 03:25:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: Write a deprecation notice in this file
Message-ID: <57ab0103.82ddc20a.5e0f1.8b48@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86129:9efa812a5f35
Date: 2016-08-10 12:24 +0200
http://bitbucket.org/pypy/pypy/changeset/9efa812a5f35/

Log:	Write a deprecation notice in this file

diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {

From pypy.commits at gmail.com  Wed Aug 10 10:11:38 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 10 Aug 2016 07:11:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Fix test_ast to check for correct
 'arguments'-fields in test_fields
Message-ID: <57ab361a.274fc20a.a9df1.ef38@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86130:fffb365644a3
Date: 2016-08-10 16:10 +0200
http://bitbucket.org/pypy/pypy/changeset/fffb365644a3/

Log:	Fix test_ast to check for correct 'arguments'-fields in test_fields

diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -45,8 +45,8 @@
         w_fields = space.getattr(ast.get(space).w_arguments,
                                  space.wrap("_fields"))
         assert space.eq_w(w_fields, space.wrap(
-            ('args', 'vararg', 'varargannotation', 'kwonlyargs', 'kwarg',
-             'kwargannotation', 'defaults', 'kw_defaults')))
+            ('args', 'vararg', 'kwonlyargs', 'kw_defaults',
+             'kwarg', 'defaults')))
         
     def test_attributes(self, space):
         w_attrs = space.getattr(ast.get(space).w_FunctionDef,

From pypy.commits at gmail.com  Wed Aug 10 12:29:10 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 10 Aug 2016 09:29:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: some more translation issues that
 showed up
Message-ID: <57ab5656.45c8c20a.3d264.84b3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async
Changeset: r86131:663016b57d78
Date: 2016-08-10 18:27 +0200
http://bitbucket.org/pypy/pypy/changeset/663016b57d78/

Log:	some more translation issues that showed up

diff --git a/pypy/interpreter/astcompiler/assemble.py.orig b/pypy/interpreter/astcompiler/assemble.py.orig
deleted file mode 100644
--- a/pypy/interpreter/astcompiler/assemble.py.orig
+++ /dev/null
@@ -1,765 +0,0 @@
-"""Python control flow graph generation and bytecode assembly."""
-
-import os
-from rpython.rlib import rfloat
-from rpython.rlib.objectmodel import specialize, we_are_translated
-
-from pypy.interpreter.astcompiler import ast, consts, misc, symtable
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.pycode import PyCode
-from pypy.tool import stdlib_opcode as ops
-
-
-class StackDepthComputationError(Exception):
-    pass
-
-
-class Instruction(object):
-    """Represents a single opcode."""
-
-    def __init__(self, opcode, arg=0):
-        self.opcode = opcode
-        self.arg = arg
-        self.lineno = 0
-        self.has_jump = False
-
-    def size(self):
-        """Return the size of bytes of this instruction when it is
-        encoded.
-        """
-        if self.opcode >= ops.HAVE_ARGUMENT:
-            return (6 if self.arg > 0xFFFF else 3)
-        return 1
-
-    def jump_to(self, target, absolute=False):
-        """Indicate the target this jump instruction.
-
-        The opcode must be a JUMP opcode.
-        """
-        self.jump = (target, absolute)
-        self.has_jump = True
-
-    def __repr__(self):
-        data = [ops.opname[self.opcode]]
-        template = "<%s"
-        if self.opcode >= ops.HAVE_ARGUMENT:
-            data.append(self.arg)
-            template += " %i"
-            if self.has_jump:
-                data.append(self.jump[0])
-                template += " %s"
-        template += ">"
-        return template % tuple(data)
-
-
-class Block(object):
-    """A basic control flow block.
-
-    It has one entry point and several possible exit points.  Its
-    instructions may be jumps to other blocks, or if control flow
-    reaches the end of the block, it continues to next_block.
-    """
-
-    marked = False
-    have_return = False
-    auto_inserted_return = False
-
-    def __init__(self):
-        self.instructions = []
-        self.next_block = None
-
-    def _post_order_see(self, stack, nextblock):
-        if nextblock.marked == 0:
-            nextblock.marked = 1
-            stack.append(nextblock)
-
-    def post_order(self):
-        """Return this block and its children in post order.  This means
-        that the graph of blocks is first cleaned up to ignore
-        back-edges, thus turning it into a DAG.  Then the DAG is
-        linearized.  For example:
-
-                   A --> B -\           =>     [A, D, B, C]
-                     \-> D ---> C
-        """
-        resultblocks = []
-        stack = [self]
-        self.marked = 1
-        while stack:
-            current = stack[-1]
-            if current.marked == 1:
-                current.marked = 2
-                if current.next_block is not None:
-                    self._post_order_see(stack, current.next_block)
-            else:
-                i = current.marked - 2
-                assert i >= 0
-                while i < len(current.instructions):
-                    instr = current.instructions[i]
-                    i += 1
-                    if instr.has_jump:
-                        current.marked = i + 2
-                        self._post_order_see(stack, instr.jump[0])
-                        break
-                else:
-                    resultblocks.append(current)
-                    stack.pop()
-        resultblocks.reverse()
-        return resultblocks
-
-    def code_size(self):
-        """Return the encoded size of all the instructions in this
-        block.
-        """
-        i = 0
-        for instr in self.instructions:
-            i += instr.size()
-        return i
-
-    def get_code(self):
-        """Encode the instructions in this block into bytecode."""
-        code = []
-        for instr in self.instructions:
-            opcode = instr.opcode
-            if opcode >= ops.HAVE_ARGUMENT:
-                arg = instr.arg
-                if instr.arg > 0xFFFF:
-                    ext = arg >> 16
-                    code.append(chr(ops.EXTENDED_ARG))
-                    code.append(chr(ext & 0xFF))
-                    code.append(chr(ext >> 8))
-                    arg &= 0xFFFF
-                code.append(chr(opcode))
-                code.append(chr(arg & 0xFF))
-                code.append(chr(arg >> 8))
-            else:
-                code.append(chr(opcode))
-        return ''.join(code)
-
-
-def _make_index_dict_filter(syms, flag):
-    i = 0
-    result = {}
-    for name, scope in syms.iteritems():
-        if scope == flag:
-            result[name] = i
-            i += 1
-    return result
-
-
- at specialize.argtype(0)
-def _iter_to_dict(iterable, offset=0):
-    result = {}
-    index = offset
-    for item in iterable:
-        result[item] = index
-        index += 1
-    return result
-
-
-class PythonCodeMaker(ast.ASTVisitor):
-    """Knows how to assemble a PyCode object."""
-
-    def __init__(self, space, name, first_lineno, scope, compile_info):
-        self.space = space
-        self.name = name
-        self.first_lineno = first_lineno
-        self.compile_info = compile_info
-        self.first_block = self.new_block()
-        self.use_block(self.first_block)
-        self.names = {}
-        self.var_names = _iter_to_dict(scope.varnames)
-        self.cell_vars = _make_index_dict_filter(scope.symbols,
-                                                 symtable.SCOPE_CELL)
-        self.free_vars = _iter_to_dict(scope.free_vars, len(self.cell_vars))
-        self.w_consts = space.newdict()
-        self.argcount = 0
-        self.kwonlyargcount = 0
-        self.lineno_set = False
-        self.lineno = 0
-        self.add_none_to_final_return = True
-
-    def new_block(self):
-        return Block()
-
-    def use_block(self, block):
-        """Start emitting bytecode into block."""
-        self.current_block = block
-        self.instrs = block.instructions
-
-    def use_next_block(self, block=None):
-        """Set this block as the next_block for the last and use it."""
-        if block is None:
-            block = self.new_block()
-        self.current_block.next_block = block
-        self.use_block(block)
-        return block
-
-    def is_dead_code(self):
-        """Return False if any code can be meaningfully added to the
-        current block, or True if it would be dead code."""
-        # currently only True after a RETURN_VALUE.
-        return self.current_block.have_return
-
-    def emit_op(self, op):
-        """Emit an opcode without an argument."""
-        instr = Instruction(op)
-        if not self.lineno_set:
-            instr.lineno = self.lineno
-            self.lineno_set = True
-        if not self.is_dead_code():
-            self.instrs.append(instr)
-            if op == ops.RETURN_VALUE:
-                self.current_block.have_return = True
-        return instr
-
-    def emit_op_arg(self, op, arg):
-        """Emit an opcode with an integer argument."""
-        instr = Instruction(op, arg)
-        if not self.lineno_set:
-            instr.lineno = self.lineno
-            self.lineno_set = True
-        if not self.is_dead_code():
-            self.instrs.append(instr)
-
-    def emit_op_name(self, op, container, name):
-        """Emit an opcode referencing a name."""
-        self.emit_op_arg(op, self.add_name(container, name))
-
-    def emit_jump(self, op, block_to, absolute=False):
-        """Emit a jump opcode to another block."""
-        self.emit_op(op).jump_to(block_to, absolute)
-
-    def add_name(self, container, name):
-        """Get the index of a name in container."""
-        name = self.scope.mangle(name)
-        try:
-            index = container[name]
-        except KeyError:
-            index = len(container)
-            container[name] = index
-        return index
-
-    def add_const(self, obj):
-        """Add a W_Root to the constant array and return its location."""
-        space = self.space
-        # To avoid confusing equal but separate types, we hash store the type
-        # of the constant in the dictionary.  Moreover, we have to keep the
-        # difference between -0.0 and 0.0 floats, and this recursively in
-        # tuples.
-        w_key = self._make_key(obj)
-
-        w_len = space.finditem(self.w_consts, w_key)
-        if w_len is None:
-            w_len = space.len(self.w_consts)
-            space.setitem(self.w_consts, w_key, w_len)
-        if space.int_w(w_len) == 0:
-            self.scope.doc_removable = False
-        return space.int_w(w_len)
-
-    def _make_key(self, obj):
-        # see the tests 'test_zeros_not_mixed*' in ../test/test_compiler.py
-        space = self.space
-        w_type = space.type(obj)
-        if space.is_w(w_type, space.w_float):
-            val = space.float_w(obj)
-            if val == 0.0 and rfloat.copysign(1., val) < 0:
-                w_key = space.newtuple([obj, space.w_float, space.w_None])
-            else:
-                w_key = space.newtuple([obj, space.w_float])
-        elif space.is_w(w_type, space.w_complex):
-            w_real = space.getattr(obj, space.wrap("real"))
-            w_imag = space.getattr(obj, space.wrap("imag"))
-            real = space.float_w(w_real)
-            imag = space.float_w(w_imag)
-            real_negzero = (real == 0.0 and
-                            rfloat.copysign(1., real) < 0)
-            imag_negzero = (imag == 0.0 and
-                            rfloat.copysign(1., imag) < 0)
-            if real_negzero and imag_negzero:
-                tup = [obj, space.w_complex, space.w_None, space.w_None,
-                       space.w_None]
-            elif imag_negzero:
-                tup = [obj, space.w_complex, space.w_None, space.w_None]
-            elif real_negzero:
-                tup = [obj, space.w_complex, space.w_None]
-            else:
-                tup = [obj, space.w_complex]
-            w_key = space.newtuple(tup)
-        elif space.is_w(w_type, space.w_tuple):
-            result_w = [obj, w_type]
-            for w_item in space.fixedview(obj):
-                result_w.append(self._make_key(w_item))
-            w_key = space.newtuple(result_w[:])
-        elif isinstance(obj, PyCode):
-            w_key = space.newtuple([obj, w_type, space.id(obj)])
-        else:
-            w_key = space.newtuple([obj, w_type])
-        return w_key
-
-    def load_const(self, obj):
-        index = self.add_const(obj)
-        self.emit_op_arg(ops.LOAD_CONST, index)
-
-    def update_position(self, lineno, force=False):
-        """Possibly change the lineno for the next instructions."""
-        if force or lineno > self.lineno:
-            self.lineno = lineno
-            self.lineno_set = False
-
-    def _resolve_block_targets(self, blocks):
-        """Compute the arguments of jump instructions."""
-        last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG
-        # extends the bytecode size, so it might invalidate the offsets
-        # we've already given.  Thus we have to loop until the number of
-        # extended args is stable.  Any extended jump at all is
-        # extremely rare, so performance is not too concerning.
-        while True:
-            extended_arg_count = 0
-            offset = 0
-            force_redo = False
-            # Calculate the code offset of each block.
-            for block in blocks:
-                block.offset = offset
-                offset += block.code_size()
-            for block in blocks:
-                offset = block.offset
-                for instr in block.instructions:
-                    offset += instr.size()
-                    if instr.has_jump:
-                        target, absolute = instr.jump
-                        op = instr.opcode
-                        # Optimize an unconditional jump going to another
-                        # unconditional jump.
-                        if op == ops.JUMP_ABSOLUTE or op == ops.JUMP_FORWARD:
-                            if target.instructions:
-                                target_op = target.instructions[0].opcode
-                                if target_op == ops.JUMP_ABSOLUTE:
-                                    target = target.instructions[0].jump[0]
-                                    instr.opcode = ops.JUMP_ABSOLUTE
-                                    absolute = True
-                                elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into
-                                    # just a RETURN
-                                    instr.opcode = ops.RETURN_VALUE
-                                    instr.arg = 0
-                                    instr.has_jump = False
-                                    # The size of the code changed,
-                                    # we have to trigger another pass
-                                    force_redo = True
-                                    continue
-                        if absolute:
-                            jump_arg = target.offset
-                        else:
-                            jump_arg = target.offset - offset
-                        instr.arg = jump_arg
-                        if jump_arg > 0xFFFF:
-                            extended_arg_count += 1
-            if (extended_arg_count == last_extended_arg_count and
-                not force_redo):
-                break
-            else:
-                last_extended_arg_count = extended_arg_count
-
-    def _build_consts_array(self):
-        """Turn the applevel constants dictionary into a list."""
-        w_consts = self.w_consts
-        space = self.space
-        consts_w = [space.w_None] * space.len_w(w_consts)
-        w_iter = space.iter(w_consts)
-        first = space.wrap(0)
-        while True:
-            try:
-                w_key = space.next(w_iter)
-            except OperationError as e:
-                if not e.match(space, space.w_StopIteration):
-                    raise
-                break
-            w_index = space.getitem(w_consts, w_key)
-            w_constant = space.getitem(w_key, first)
-            w_constant = misc.intern_if_common_string(space, w_constant)
-            consts_w[space.int_w(w_index)] = w_constant
-        return consts_w
-
-    def _get_code_flags(self):
-        """Get an extra flags that should be attached to the code object."""
-        raise NotImplementedError
-
-    def _stacksize(self, blocks):
-        """Compute co_stacksize."""
-        for block in blocks:
-            block.initial_depth = 0
-        # Assumes that it is sufficient to walk the blocks in 'post-order'.
-        # This means we ignore all back-edges, but apart from that, we only
-        # look into a block when all the previous blocks have been done.
-        self._max_depth = 0
-        for block in blocks:
-            depth = self._do_stack_depth_walk(block)
-            if block.auto_inserted_return and depth != 0:
-                os.write(2, "StackDepthComputationError in %s at %s:%s\n" % (
-                    self.compile_info.filename, self.name, self.first_lineno))
-                raise StackDepthComputationError   # fatal error
-        return self._max_depth
-
-    def _next_stack_depth_walk(self, nextblock, depth):
-        if depth > nextblock.initial_depth:
-            nextblock.initial_depth = depth
-
-    def _do_stack_depth_walk(self, block):
-        depth = block.initial_depth
-        for instr in block.instructions:
-            depth += _opcode_stack_effect(instr.opcode, instr.arg)
-            if depth >= self._max_depth:
-                self._max_depth = depth
-            jump_op = instr.opcode
-            if instr.has_jump:
-                target_depth = depth
-                if jump_op == ops.FOR_ITER:
-                    target_depth -= 2
-                elif (jump_op == ops.SETUP_FINALLY or
-                      jump_op == ops.SETUP_EXCEPT or
-                      jump_op == ops.SETUP_WITH):
-                    if jump_op == ops.SETUP_FINALLY:
-                        target_depth += 4
-                    elif jump_op == ops.SETUP_EXCEPT:
-                        target_depth += 4
-                    elif jump_op == ops.SETUP_WITH:
-                        target_depth += 3
-                    if target_depth > self._max_depth:
-                        self._max_depth = target_depth
-                elif (jump_op == ops.JUMP_IF_TRUE_OR_POP or
-                      jump_op == ops.JUMP_IF_FALSE_OR_POP):
-                    depth -= 1
-                self._next_stack_depth_walk(instr.jump[0], target_depth)
-                if jump_op == ops.JUMP_ABSOLUTE or jump_op == ops.JUMP_FORWARD:
-                    # Nothing more can occur.
-                    break
-            elif jump_op == ops.RETURN_VALUE or jump_op == ops.RAISE_VARARGS:
-                # Nothing more can occur.
-                break
-        else:
-            if block.next_block:
-                self._next_stack_depth_walk(block.next_block, depth)
-        return depth
-
-    def _build_lnotab(self, blocks):
-        """Build the line number table for tracebacks and tracing."""
-        current_line = self.first_lineno
-        current_off = 0
-        table = []
-        push = table.append
-        for block in blocks:
-            offset = block.offset
-            for instr in block.instructions:
-                if instr.lineno:
-                    # compute deltas
-                    line = instr.lineno - current_line
-                    if line < 0:
-                        continue
-                    addr = offset - current_off
-                    # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned
-                    # char).  Depending on when SET_LINENO instructions
-                    # are emitted this is not always true.  Consider the
-                    # code:
-                    #     a = (1,
-                    #          b)
-                    # In the bytecode stream, the assignment to "a"
-                    # occurs after the loading of "b".  This works with
-                    # the C Python compiler because it only generates a
-                    # SET_LINENO instruction for the assignment.
-                    if line or addr:
-                        while addr > 255:
-                            push(chr(255))
-                            push(chr(0))
-                            addr -= 255
-                        while line > 255:
-                            push(chr(addr))
-                            push(chr(255))
-                            line -= 255
-                            addr = 0
-                        push(chr(addr))
-                        push(chr(line))
-                        current_line = instr.lineno
-                        current_off = offset
-                offset += instr.size()
-        return ''.join(table)
-
-    def assemble(self):
-        """Build a PyCode object."""
-        # Unless it's interactive, every code object must end in a return.
-        if not self.current_block.have_return:
-            self.use_next_block()
-            if self.add_none_to_final_return:
-                self.load_const(self.space.w_None)
-            self.emit_op(ops.RETURN_VALUE)
-            self.current_block.auto_inserted_return = True
-        # Set the first lineno if it is not already explicitly set.
-        if self.first_lineno == -1:
-            if self.first_block.instructions:
-                self.first_lineno = self.first_block.instructions[0].lineno
-            else:
-                self.first_lineno = 1
-        blocks = self.first_block.post_order()
-        self._resolve_block_targets(blocks)
-        lnotab = self._build_lnotab(blocks)
-        stack_depth = self._stacksize(blocks)
-        consts_w = self._build_consts_array()
-        names = _list_from_dict(self.names)
-        var_names = _list_from_dict(self.var_names)
-        cell_names = _list_from_dict(self.cell_vars)
-        free_names = _list_from_dict(self.free_vars, len(cell_names))
-        flags = self._get_code_flags()
-        # (Only) inherit compilerflags in PyCF_MASK
-        flags |= (self.compile_info.flags & consts.PyCF_MASK)
-        bytecode = ''.join([block.get_code() for block in blocks])
-        return PyCode(self.space,
-                      self.argcount,
-                      self.kwonlyargcount,
-                      len(self.var_names),
-                      stack_depth,
-                      flags,
-                      bytecode,
-                      list(consts_w),
-                      names,
-                      var_names,
-                      self.compile_info.filename,
-                      self.name,
-                      self.first_lineno,
-                      lnotab,
-                      free_names,
-                      cell_names,
-                      self.compile_info.hidden_applevel)
-
-
-def _list_from_dict(d, offset=0):
-    result = [None] * len(d)
-    for obj, index in d.iteritems():
-        result[index - offset] = obj
-    return result
-
-
-_static_opcode_stack_effects = {
-    ops.NOP: 0,
-
-    ops.POP_TOP: -1,
-    ops.ROT_TWO: 0,
-    ops.ROT_THREE: 0,
-    ops.DUP_TOP: 1,
-    ops.DUP_TOP_TWO: 2,
-
-    ops.UNARY_POSITIVE: 0,
-    ops.UNARY_NEGATIVE: 0,
-    ops.UNARY_NOT: 0,
-    ops.UNARY_INVERT: 0,
-
-    ops.LIST_APPEND: -1,
-    ops.SET_ADD: -1,
-    ops.MAP_ADD: -2,
-<<<<<<< local
-=======
-    # XXX 
-    ops.STORE_MAP: -2,
->>>>>>> other
-
-    ops.BINARY_POWER: -1,
-    ops.BINARY_MULTIPLY: -1,
-    ops.BINARY_MODULO: -1,
-    ops.BINARY_ADD: -1,
-    ops.BINARY_SUBTRACT: -1,
-    ops.BINARY_SUBSCR: -1,
-    ops.BINARY_FLOOR_DIVIDE: -1,
-    ops.BINARY_TRUE_DIVIDE: -1,
-    ops.BINARY_MATRIX_MULTIPLY: -1,
-    ops.BINARY_LSHIFT: -1,
-    ops.BINARY_RSHIFT: -1,
-    ops.BINARY_AND: -1,
-    ops.BINARY_OR: -1,
-    ops.BINARY_XOR: -1,
-
-    ops.INPLACE_FLOOR_DIVIDE: -1,
-    ops.INPLACE_TRUE_DIVIDE: -1,
-    ops.INPLACE_ADD: -1,
-    ops.INPLACE_SUBTRACT: -1,
-    ops.INPLACE_MULTIPLY: -1,
-    ops.INPLACE_MODULO: -1,
-    ops.INPLACE_POWER: -1,
-    ops.INPLACE_MATRIX_MULTIPLY: -1,
-    ops.INPLACE_LSHIFT: -1,
-    ops.INPLACE_RSHIFT: -1,
-    ops.INPLACE_AND: -1,
-    ops.INPLACE_OR: -1,
-    ops.INPLACE_XOR: -1,
-
-    ops.STORE_SUBSCR: -3,
-    ops.DELETE_SUBSCR: -2,
-
-    ops.GET_ITER: 0,
-    ops.FOR_ITER: 1,
-    ops.BREAK_LOOP: 0,
-    ops.CONTINUE_LOOP: 0,
-    ops.SETUP_LOOP: 0,
-
-    ops.PRINT_EXPR: -1,
-
-<<<<<<< local
-    ops.WITH_CLEANUP_START: -1,
-    ops.WITH_CLEANUP_FINISH: -1,  # XXX Sometimes more
-=======
-    # TODO 
-    ops.WITH_CLEANUP: -1,
->>>>>>> other
-    ops.LOAD_BUILD_CLASS: 1,
-<<<<<<< local
-=======
-    # TODO 
-    ops.STORE_LOCALS: -1,
->>>>>>> other
-    ops.POP_BLOCK: 0,
-    ops.POP_EXCEPT: -1,
-    ops.END_FINALLY: -4,     # assume always 4: we pretend that SETUP_FINALLY
-                             # pushes 4.  In truth, it would only push 1 and
-                             # the corresponding END_FINALLY only pops 1.
-    ops.SETUP_WITH: 1,
-    ops.SETUP_FINALLY: 0,
-    ops.SETUP_EXCEPT: 0,
-
-    ops.RETURN_VALUE: -1,
-    ops.YIELD_VALUE: 0,
-    ops.YIELD_FROM: -1,
-    ops.COMPARE_OP: -1,
-
-    # TODO 
-    ops.LOOKUP_METHOD: 1,
-
-    ops.LOAD_NAME: 1,
-    ops.STORE_NAME: -1,
-    ops.DELETE_NAME: 0,
-
-    ops.LOAD_FAST: 1,
-    ops.STORE_FAST: -1,
-    ops.DELETE_FAST: 0,
-
-    ops.LOAD_ATTR: 0,
-    ops.STORE_ATTR: -2,
-    ops.DELETE_ATTR: -1,
-
-    ops.LOAD_GLOBAL: 1,
-    ops.STORE_GLOBAL: -1,
-    ops.DELETE_GLOBAL: 0,
-    ops.DELETE_DEREF: 0,
-
-    ops.LOAD_CLOSURE: 1,
-    ops.LOAD_DEREF: 1,
-    ops.STORE_DEREF: -1,
-    ops.DELETE_DEREF: 0,
-
-    ops.LOAD_CONST: 1,
-
-    ops.IMPORT_STAR: -1,
-    ops.IMPORT_NAME: -1,
-    ops.IMPORT_FROM: 1,
-
-    ops.JUMP_FORWARD: 0,
-    ops.JUMP_ABSOLUTE: 0,
-    ops.JUMP_IF_TRUE_OR_POP: 0,
-    ops.JUMP_IF_FALSE_OR_POP: 0,
-    ops.POP_JUMP_IF_TRUE: -1,
-    ops.POP_JUMP_IF_FALSE: -1,
-    # TODO 
-    ops.JUMP_IF_NOT_DEBUG: 0,
-
-    # TODO 
-    ops.BUILD_LIST_FROM_ARG: 1,
-}
-
-
-def _compute_UNPACK_SEQUENCE(arg):
-    return arg - 1
-
-def _compute_UNPACK_EX(arg):
-    return (arg & 0xFF) + (arg >> 8)
-
-def _compute_BUILD_TUPLE(arg):
-    return 1 - arg
-
-def _compute_BUILD_LIST(arg):
-    return 1 - arg
-
-def _compute_BUILD_SET(arg):
-    return 1 - arg
-
-def _compute_BUILD_MAP(arg):
-    return 1 - 2 * arg
-
-def _compute_BUILD_MAP_UNPACK(arg):
-    return 1 - arg
-
-def _compute_MAKE_CLOSURE(arg):
-    return -2 - _num_args(arg) - ((arg >> 16) & 0xFFFF)
-
-def _compute_MAKE_FUNCTION(arg):
-    return -1 - _num_args(arg) - ((arg >> 16) & 0xFFFF)
-
-def _compute_BUILD_SLICE(arg):
-    if arg == 3:
-        return -2
-    else:
-        return -1
-
-def _compute_RAISE_VARARGS(arg):
-    return -arg
-
-def _num_args(oparg):
-    return (oparg % 256) + 2 * ((oparg // 256) % 256)
-
-def _compute_CALL_FUNCTION(arg):
-    return -_num_args(arg)
-
-def _compute_CALL_FUNCTION_VAR(arg):
-    return -_num_args(arg) - 1
-
-def _compute_CALL_FUNCTION_KW(arg):
-    return -_num_args(arg) - 1
-
-def _compute_CALL_FUNCTION_VAR_KW(arg):
-    return -_num_args(arg) - 2
-
-def _compute_CALL_METHOD(arg):
-    return -_num_args(arg) - 1
-
-
-_stack_effect_computers = {}
-for name, func in globals().items():
-    if name.startswith("_compute_"):
-        func._always_inline_ = True
-        _stack_effect_computers[getattr(ops, name[9:])] = func
-for op, value in _static_opcode_stack_effects.iteritems():
-    def func(arg, _value=value):
-        return _value
-    func._always_inline_ = True
-    _stack_effect_computers[op] = func
-del name, func, op, value
-
-
-def _opcode_stack_effect(op, arg):
-    """Return the stack effect of a opcode an its argument."""
-    if we_are_translated():
-        for possible_op in ops.unrolling_opcode_descs:
-            # EXTENDED_ARG should never get in here.
-            if possible_op.index == ops.EXTENDED_ARG:
-                continue
-            if op == possible_op.index:
-                return _stack_effect_computers[possible_op.index](arg)
-        else:
-            raise AssertionError("unknown opcode: %s" % (op,))
-    else:
-        try:
-            return _static_opcode_stack_effects[op]
-        except KeyError:
-            try:
-                return _stack_effect_computers[op](arg)
-            except KeyError:
-                raise KeyError("Unknown stack effect for %s (%s)" %
-                               (ops.opname[op], op))
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.pyparser.pygram import syms, tokens
 from pypy.interpreter.pyparser.error import SyntaxError
 from pypy.interpreter.pyparser import parsestring
+from rpython.rlib.objectmodel import always_inline
 
 
 def ast_from_node(space, node, compile_info):
@@ -1169,6 +1170,7 @@
                 raise
             return self.space.call_function(self.space.w_float, w_num_str)
 
+    @always_inline
     def handle_dictelement(self, node, i):
         if node.get_child(i).type == tokens.DOUBLESTAR:
             key = None
@@ -1178,7 +1180,7 @@
             key = self.handle_expr(node.get_child(i))
             value = self.handle_expr(node.get_child(i+2))
             i += 3
-        return [i,key,value]
+        return (i,key,value)
     
     def handle_atom(self, atom_node):
         first_child = atom_node.get_child(0)
@@ -1374,10 +1376,7 @@
                            set_maker.get_column())
 
     def handle_dictcomp(self, dict_maker):
-        dictelement = self.handle_dictelement(dict_maker, 0)
-        i = dictelement[0]
-        key = dictelement[1]
-        value = dictelement[2]
+        i, key, value = self.handle_dictelement(dict_maker, 0)
         comps = self.comprehension_helper(dict_maker.get_child(i))
         return ast.DictComp(key, value, comps, dict_maker.get_lineno(),
                             dict_maker.get_column())
@@ -1387,10 +1386,9 @@
         values = []
         i = 0
         while i < node.num_children():
-            dictelement = self.handle_dictelement(node, i)
-            i = dictelement[0]
-            keys.append(dictelement[1])
-            values.append(dictelement[2])
+            i, key, value = self.handle_dictelement(node, i)
+            keys.append(key)
+            values.append(value)
             i += 1
         return ast.Dict(keys, values, node.get_lineno(), node.get_column())
     
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -441,8 +441,7 @@
         # 4. load class name
         self.load_const(self.space.wrap(cls.name.decode('utf-8')))
         # 5. generate the rest of the code for the call
-        self._make_call(2,
-                        cls.bases, cls.keywords)
+        self._make_call(2, cls.bases, cls.keywords)
         # 6. apply decorators
         if cls.decorator_list:
             for i in range(len(cls.decorator_list)):
@@ -1348,8 +1347,7 @@
         if self._optimize_method_call(call):
             return
         call.func.walkabout(self)
-        self._make_call(0,
-                        call.args, call.keywords)
+        self._make_call(0, call.args, call.keywords)
     
     def _call_has_no_star_args(self, call):
         if call.args is not None:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1056,7 +1056,7 @@
         w_value = self.popvalue()
         w_gen = self.peekvalue()
         if isinstance(w_gen, Coroutine):
-            if (w_gen.descr_gi_code(w_gen).co_flags & consts.CO_COROUTINE and
+            if (w_gen.descr_gi_code(space).co_flags & consts.CO_COROUTINE and
                not self.pycode.co_flags & (consts.CO_COROUTINE |
                                        consts.CO_ITERABLE_COROUTINE)):
                 raise oefmt(self.space.w_TypeError,
@@ -1479,27 +1479,29 @@
         self.pushvalue(res)
     
     def BEFORE_ASYNC_WITH(self, oparg, next_instr):
+        space = self.space
         w_manager = self.peekvalue()
-        w_enter = self.space.lookup(w_manager, "__aenter__")
-        w_descr = self.space.lookup(w_manager, "__aexit__")
+        w_enter = space.lookup(w_manager, "__aenter__")
+        w_descr = space.lookup(w_manager, "__aexit__")
         if w_enter is None or w_descr is None:
-            raise oefmt(self.space.w_AttributeError,
+            raise oefmt(space.w_AttributeError,
                         "'%T' object is not a context manager (no __aenter__/"
                         "__aexit__ method)", w_manager)
-        w_exit = self.space.get(w_descr, w_manager)
+        w_exit = space.get(w_descr, w_manager)
         self.settopvalue(w_exit)
-        w_result = self.space.get_and_call_function(w_enter, w_manager)
+        w_result = space.get_and_call_function(w_enter, w_manager)
         self.pushvalue(w_result)
     
     def GET_AITER(self, oparg, next_instr):
+        space = self.space
         w_obj = self.peekvalue()
-        w_func = self.space.lookup(w_obj, "__aiter__")
+        w_func = space.lookup(w_obj, "__aiter__")
         if w_func is None:
             raise oefmt(space.w_AttributeError,
                         "object %T does not have __aiter__ method",
                         w_obj)
         w_iter = space.get_and_call_function(w_func, w_obj)
-        w_awaitable = w_iter._GetAwaitableIter(self.space)
+        w_awaitable = w_iter._GetAwaitableIter(space)
         if w_awaitable is None:
             raise oefmt(space.w_TypeError,
                         "'async for' received an invalid object "
@@ -1507,14 +1509,15 @@
         self.settopvalue(w_awaitable)
     
     def GET_ANEXT(self, oparg, next_instr):
+        space = self.space
         w_aiter = self.peekvalue()
-        w_func = self.space.lookup(w_aiter, "__anext__")
+        w_func = space.lookup(w_aiter, "__anext__")
         if w_func is None:
             raise oefmt(space.w_AttributeError,
                         "object %T does not have __anext__ method",
                         w_aiter)
         w_next_iter = space.get_and_call_function(w_func, w_aiter)
-        w_awaitable = w_next_iter._GetAwaitableIter(self.space)
+        w_awaitable = w_next_iter._GetAwaitableIter(space)
         if w_awaitable is None:
             raise oefmt(space.w_TypeError,
                         "'async for' received an invalid object "
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -85,7 +85,7 @@
     def iteratekeys(self, space):
         return space.iter(self.keys(space))
 
-    def itervalues(self, space):
+    def iteratevalues(self, space):
         return space.iter(self.values(space))
 
     def iteritems(self, space):
@@ -112,7 +112,7 @@
     keys = interp2app(W_ZipCache.keys),
     iterkeys = interp2app(W_ZipCache.iteratekeys),
     values = interp2app(W_ZipCache.values),
-    itervalues = interp2app(W_ZipCache.itervalues),
+    itervalues = interp2app(W_ZipCache.iteratevalues),
     clear = interp2app(W_ZipCache.clear),
     __delitem__ = interp2app(W_ZipCache.delitem),
 )

From pypy.commits at gmail.com  Thu Aug 11 03:54:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 00:54:48 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Correctly close the remote end
 of the socketpair. Otherwise, the
Message-ID: <57ac2f48.45c8c20a.dc13a.67fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86132:5797950d42c5
Date: 2016-08-11 09:53 +0200
http://bitbucket.org/pypy/pypy/changeset/5797950d42c5/

Log:	Correctly close the remote end of the socketpair. Otherwise, the
	subprocess may remain alive forever, trying to read from its own end
	of the socketpair, because it also happens to keep the remote end
	open.

diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -228,7 +228,7 @@
         s1, s2 = socket.socketpair()
         initial_subproc = subprocess.Popen(
             [executable, '--revdb-replay', revdb_log_filename,
-             str(s2.fileno())])
+             str(s2.fileno())], preexec_fn=s1.close)
         s2.close()
         child = ReplayProcess(initial_subproc.pid, s1,
                               linecacheoutput=linecacheoutput)
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -263,7 +263,7 @@
         s1, s2 = socket.socketpair()
         subproc = subprocess.Popen(
             [str(self.exename), '--revdb-replay', str(self.rdbname),
-             str(s2.fileno())], **kwds)
+             str(s2.fileno())], preexec_fn=s1.close, **kwds)
         s2.close()
         self.subproc = subproc
         child = ReplayProcess(subproc.pid, s1)

From pypy.commits at gmail.com  Thu Aug 11 03:54:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 00:54:50 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: in-progress
Message-ID: <57ac2f4a.c75dc20a.853ec.6920@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86133:710590b97b42
Date: 2016-08-11 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/710590b97b42/

Log:	in-progress

diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c
--- a/rpython/translator/c/src/entrypoint.c
+++ b/rpython/translator/c/src/entrypoint.c
@@ -101,10 +101,10 @@
 
     RPython_StartupCode();
 
+#ifndef RPY_REVERSE_DEBUGGER
     exitcode = STANDALONE_ENTRY_POINT(argc, argv);
-
-#ifdef RPY_REVERSE_DEBUGGER
-    rpy_reverse_db_teardown();
+#else
+    exitcode = rpy_reverse_db_main(STANDALONE_ENTRY_POINT, argc, argv);
 #endif
 
     pypy_debug_alloc_results();
diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py
--- a/rpython/translator/revdb/gencsupp.py
+++ b/rpython/translator/revdb/gencsupp.py
@@ -84,16 +84,16 @@
                            # that the calls should really be done
     #
     # hack: we don't need the flag for at least this common function
+    if call_code == 'RPyGilAcquire();':
+        return 'RPY_REVDB_CALL_GILCTRL(%s);' % (call_code,)
     if call_code == 'RPyGilRelease();':
-        return 'RPY_REVDB_CALL_GILCTRL(%s);' % (call_code,)
-    if call_code == 'RPyGilAcquire();':
         # Could also work with a regular RPY_REVDB_CALL_VOID, but we
         # use a different byte (0xFD instead of 0xFC) to detect more
         # sync misses.  In a single-threaded environment this 0xFD
         # byte is not needed at all, but in a multi-threaded
-        # environment it ensures that during replaying, we don't go
-        # past the RPyGilAcquire() in case a different thread must run
-        # next.
+        # environment it ensures that during replaying, just after
+        # reading the 0xFD, we switch to a different thread if needed
+        # (actually implemented with stacklets).
         return 'RPY_REVDB_CALL_GIL(%s);' % (call_code,)
     #
     tp = funcgen.lltypename(v_result)
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -44,9 +44,10 @@
 
 typedef struct {
     Signed version;
-    uint64_t reserved1, reserved2;
+    uint64_t main_thread_id;
+    uint64_t reserved2;
     void *ptr1, *ptr2;
-    int reversed3;
+    int reserved3;
     int argc;
     char **argv;
 } rdb_header_t;
@@ -130,8 +131,7 @@
     _RPY_REVDB_UNLOCK();
 }
 
-RPY_EXTERN
-void rpy_reverse_db_teardown(void)
+static void reverse_db_teardown(void)
 {
     uint64_t stop_points;
     if (!RPY_RDB_REPLAY) {
@@ -237,6 +237,7 @@
         h.ptr2 = &rpy_revdb;
         h.argc = argc;
         h.argv = argv;
+        h.main_thread_id = (uint64_t)pthread_self();
         write_all((const char *)&h, sizeof(h));
 
         /* write the whole content of rpy_rdb_struct */
@@ -254,7 +255,7 @@
     rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32;
     rpy_revdb.unique_id_seen = 1;
 
-    rpy_active_thread = 0;  /* write an ASYNC_THREAD_SWITCH first in the log */
+    rpy_active_thread = 1;
     rpy_active_thread_ptr = &rpy_active_thread;
 
     pthread_atfork(NULL, NULL, close_revdb_fileno_in_fork_child);
@@ -625,6 +626,7 @@
 */
 
 #include "src-revdb/fd_recv.c"
+#include "src/stacklet/stacklet.c"   /* for replaying threads */
 
 #define INIT_VERSION_NUMBER   0xd80100
 
@@ -658,6 +660,148 @@
 static uint64_t *future_ids, *future_next_id;
 static void *finalizer_tree, *destructor_tree;
 
+static stacklet_thread_handle st_thread;
+static stacklet_handle st_outer_controller_h;
+static uint64_t current_thread_id, target_thread_id;
+static void *thread_tree_root;
+
+
+struct replay_thread_main_s {
+    Signed (*entry_point)(Signed, char **);
+    int argc;
+    char **argv;
+};
+struct replay_thread_s {
+    uint64_t tid;
+    stacklet_handle h;
+};
+
+static stacklet_handle replay_thread_main(stacklet_handle h, void *arg)
+{
+    /* main thread starts */
+    struct replay_thread_main_s *m = arg;
+    st_outer_controller_h = h;
+    m->entry_point(m->argc, m->argv);
+
+    /* main thread finished, program stops */
+    reverse_db_teardown();
+
+    /* unreachable */
+    abort();
+}
+
+static void replay_invoke_callback(unsigned char e);
+
+static stacklet_handle replay_thread_sub(stacklet_handle h, void *ignored)
+{
+    /* A non-main thread starts.  What is does is invoke a "callback",
+       which is the argument passed to rthread.ll_start_new_thread().
+       We get it here because the first thing stored in the log about
+       this thread should be a callback identifier.
+    */
+    unsigned char e1;
+    st_outer_controller_h = h;
+
+    if (rpy_revdb.buf_limit >= rpy_revdb.buf_p)
+        rpy_reverse_db_fetch(__FILE__, __LINE__);
+
+    _RPY_REVDB_EMIT_REPLAY(unsigned char _e, e1)
+    replay_invoke_callback(e1);
+
+    /* the thread finishes here.  Return to the outer controller. */
+    return st_outer_controller_h;
+}
+
+static int compare_replay_thread(const void *a, const void *b)
+{
+    uint64_t ta = ((const struct replay_thread_s *)a)->tid;
+    uint64_t tb = ((const struct replay_thread_s *)b)->tid;
+    if (ta < tb)
+        return -1;
+    if (ta == tb)
+        return 0;
+    else
+        return 1;
+}
+
+RPY_EXTERN
+int rpy_reverse_db_main(Signed entry_point(Signed, char**),
+                        int argc, char **argv)
+{
+    if (!RPY_RDB_REPLAY) {
+        int exitcode = (int)entry_point(argc, argv);
+        reverse_db_teardown();
+        return exitcode;
+    }
+    else {
+        /* start the entry point inside a new stacklet, so that we
+           can switch it away at any point later */
+        struct replay_thread_main_s m;
+        stacklet_handle h;
+        m.entry_point = entry_point;
+        m.argc = argc;
+        m.argv = argv;
+        h = stacklet_new(st_thread, replay_thread_main, &m);
+
+        /* We reach this point only if we start a second thread.  This
+           is done by revdb_switch_thread(), which switches back to
+           'st_outer_controller_h'.  This is the outer controller
+           loop.
+        */
+        attach_gdb();
+        while (1) {
+            struct replay_thread_s *node, **item, dummy;
+
+            if (h == NULL)
+                goto out_of_memory;
+
+            if (h != EMPTY_STACKLET_HANDLE) {
+                /* save 'h' as the stacklet handle for the thread
+                   'current_thread_id' */
+                node = malloc(sizeof(struct replay_thread_s));
+                if (!node)
+                    goto out_of_memory;
+                node->tid = current_thread_id;
+                node->h = h;
+                item = tsearch(node, &thread_tree_root, compare_replay_thread);
+                if (item == NULL)
+                    goto out_of_memory;
+
+                if (*item != node) {
+                    fprintf(stderr, "thread switch: duplicate thread\n");
+                    exit(1);
+                }
+            }
+            else {
+                /* current_thread_id terminated */
+            }
+
+            /* fetch out (and delete) the handle for the target thread */
+            current_thread_id = target_thread_id;
+            dummy.tid = target_thread_id;
+            item = tfind(&dummy, &thread_tree_root, compare_replay_thread);
+            if (item == NULL) {
+                /* it's a new thread, start it now */
+                h = stacklet_new(st_thread, replay_thread_sub, NULL);
+            }
+            else {
+                node = *item;
+                assert(node->tid == target_thread_id);
+                h = node->h;
+                tdelete(node, &thread_tree_root, compare_replay_thread);
+                free(node);
+
+                h = stacklet_switch(h);
+            }
+        }
+        abort(); /* unreachable */
+
+    out_of_memory:
+        fprintf(stderr, "thread switch: out of memory\n");
+        exit(1);
+    }
+}
+
 RPY_EXTERN
 void attach_gdb(void)
 {
@@ -796,6 +940,7 @@
                 (long)h.version, (long)RDB_VERSION);
         exit(1);
     }
+    current_thread_id = h.main_thread_id;
     if (h.ptr1 != &rpy_reverse_db_stop_point ||
         h.ptr2 != &rpy_revdb) {
         fprintf(stderr,
@@ -833,6 +978,7 @@
     set_revdb_breakpoints();
 
     empty_string = make_rpy_string(0);
+    st_thread = stacklet_newthread();  /* replaying doesn't use real threads */
 
     write_answer(ANSWER_INIT, INIT_VERSION_NUMBER, total_stop_points, 0);
 
@@ -887,8 +1033,6 @@
             fprintf(stderr, "bad log format: incomplete packet\n");
             exit(1);
         }
-
-     read_next_packet:
         keep = rpy_revdb.buf_readend - rpy_revdb.buf_p;
         assert(keep >= 0);
 
@@ -923,8 +1067,13 @@
                 return;
 
             case ASYNC_THREAD_SWITCH:
-                fetch_async_block();
-                goto read_next_packet;
+                target_thread_id = fetch_async_block();
+                _RPY_REVDB_PRINT("[THRD]", target_thread_id);
+                rpy_revdb.buf_limit = rpy_revdb.buf_p;
+                st_outer_controller_h = stacklet_switch(st_outer_controller_h);
+                if (rpy_revdb.buf_limit == rpy_revdb.buf_p)
+                    rpy_reverse_db_fetch(__FILE__, __LINE__);
+                return;
 
             default:
                 fprintf(stderr, "bad packet header %d\n", (int)header);
@@ -1157,7 +1306,6 @@
         memcpy(future_ids, extra, cmd->extra_size);
         future_ids[cmd->extra_size / sizeof(uint64_t)] = 0;
         uid_break = *future_ids;
-        //attach_gdb();
     }
     future_next_id = future_ids;
 }
@@ -1501,6 +1649,22 @@
     RPY_CALLBACKLOCS     /* macro from revdb_def.h */
 };
 
+static void replay_invoke_callback(unsigned char e)
+{
+    unsigned long index;
+    unsigned char e2;
+    void (*pfn)(void);
+    _RPY_REVDB_EMIT_REPLAY(unsigned char _e, e2)
+    index = (e << 8) | e2;
+    index -= 300;
+    if (index >= (sizeof(callbacklocs) / sizeof(callbacklocs[0]))) {
+        fprintf(stderr, "bad callback index %lx\n", index);
+        exit(1);
+    }
+    pfn = callbacklocs[index];
+    pfn();
+}
+
 RPY_EXTERN
 void rpy_reverse_db_invoke_callback(unsigned char e)
 {
@@ -1509,19 +1673,7 @@
        callback identifier. */
 
     do {
-        unsigned long index;
-        unsigned char e2;
-        void (*pfn)(void);
-        _RPY_REVDB_EMIT_REPLAY(unsigned char _e, e2)
-        index = (e << 8) | e2;
-        index -= 300;
-        if (index >= (sizeof(callbacklocs) / sizeof(callbacklocs[0]))) {
-            fprintf(stderr, "bad callback index\n");
-            exit(1);
-        }
-        pfn = callbacklocs[index];
-        pfn();
-
+        replay_invoke_callback(e);
         _RPY_REVDB_EMIT_REPLAY(unsigned char _e, e)
     } while (e != 0xFC);
 }
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -29,7 +29,8 @@
 /* ------------------------------------------------------------ */
 
 RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]);
-RPY_EXTERN void rpy_reverse_db_teardown(void);
+RPY_EXTERN int rpy_reverse_db_main(Signed entry_point(Signed, char**),
+                                   int argc, char **argv);
 
 /* enable to print locations to stderr of all the EMITs */
 #ifdef RPY_REVDB_PRINT_ALL
@@ -92,7 +93,7 @@
             char *_end1 = _src + sizeof(_e);                            \
             memcpy(&_e, _src, sizeof(_e));                              \
             rpy_revdb.buf_p = _end1;                                    \
-            _RPY_REVDB_PRINT("[read]", _e);                             \
+            _RPY_REVDB_PRINT("[ rd ]", _e);                             \
             if (_end1 >= rpy_revdb.buf_limit)                           \
                 rpy_reverse_db_fetch(__FILE__, __LINE__);               \
             variable = _e;                                              \
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -25,7 +25,7 @@
         assert header == 'RevDB:\t' + '\t'.join(expected_argv) + '\n\x00'
         #
         x = self.read1('P'); assert x == 0x00FF0003
-        x = self.read1('P'); assert x == 0
+        x = self.read1('P'); self.main_thread_id = x
         x = self.read1('P'); assert x == 0
         x = self.read1('P'); #assert x == &rpy_reverse_db_stop_point
         x = self.read1('P'); #assert x == &rpy_revdb
@@ -33,7 +33,6 @@
         self.argc = self.read1('i')
         self.argv = self.read1('P')
         self.current_packet_end = self.cur
-        self.main_thread_id = self.switch_thread()
         self.read_check_argv(expected_argv)
 
     def read1(self, mode):
@@ -91,16 +90,16 @@
 
     def write_call(self, expected_string):
         x = self.next()     # raw_malloc: the pointer we got
+        self.gil_release()
         self.same_stack()   # write
         x = self.next(); assert x == len(expected_string)
         self.same_stack()   # errno
         x = self.next('i'); assert x == 0      # errno
-        self.gil_acquire()
 
     def same_stack(self):
         x = self.next('c'); assert x == '\xFC'
 
-    def gil_acquire(self):
+    def gil_release(self):
         x = self.next('c'); assert x == '\xFD'
 
     def switch_thread(self, expected=None):
diff --git a/rpython/translator/revdb/test/test_callback.py b/rpython/translator/revdb/test/test_callback.py
--- a/rpython/translator/revdb/test/test_callback.py
+++ b/rpython/translator/revdb/test/test_callback.py
@@ -63,19 +63,19 @@
         self.compile(main, backendopt=False)
         out = self.run('Xx')
         rdb = self.fetch_rdb([self.exename, 'Xx'])
+        rdb.gil_release()
         rdb.same_stack()                        # callmesimple()
         x = rdb.next('i'); assert x == 55555
-        rdb.gil_acquire()
         rdb.write_call('55555\n')
+        rdb.gil_release()
         b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
         x = rdb.next('i'); assert x == 40       # arg n
-        rdb.gil_acquire()
+        rdb.gil_release()
         x = rdb.next('!h'); assert x == b       # -> callback
         x = rdb.next('i'); assert x == 3        # arg n
-        rdb.gil_acquire()
+        rdb.gil_release()
         rdb.same_stack()                        # <- return in main thread
         x = rdb.next('i'); assert x == 4000 * 300   # return from callme()
-        rdb.gil_acquire()
         rdb.write_call('%s\n' % (4000 * 300,))
         x = rdb.next('q'); assert x == 0      # number of stop points
         assert rdb.done()
@@ -85,17 +85,17 @@
         self.compile(main, backendopt=False)
         out = self.run('Xx')
         rdb = self.fetch_rdb([self.exename, 'Xx'])
+        rdb.gil_release()
         b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
         x = rdb.next('i'); assert x == 40       # arg n
-        rdb.gil_acquire()
         rdb.write_call('40\n')
+        rdb.gil_release()
         x = rdb.next('!h'); assert x == b       # -> callback again
         x = rdb.next('i'); assert x == 3        # arg n
-        rdb.gil_acquire()
         rdb.write_call('3\n')
+        rdb.gil_release()
         rdb.same_stack()                        # -> return in main thread
         x = rdb.next('i'); assert x == 120      # <- return from callme()
-        rdb.gil_acquire()
         rdb.write_call('120\n')
         x = rdb.next('q'); assert x == 2        # number of stop points
         assert rdb.done()

From pypy.commits at gmail.com  Thu Aug 11 03:58:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 00:58:30 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Forgot to add the test file.
Message-ID: <57ac3026.c15e1c0a.1a675.2c00@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86134:678a96c2a2bf
Date: 2016-08-11 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/678a96c2a2bf/

Log:	Forgot to add the test file.

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -748,7 +748,6 @@
            'st_outer_controller_h'.  This is the outer controller
            loop.
         */
-        attach_gdb();
         while (1) {
             struct replay_thread_s *node, **item, dummy;
 
diff --git a/rpython/translator/revdb/test/test_thread.py b/rpython/translator/revdb/test/test_thread.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/test/test_thread.py
@@ -0,0 +1,107 @@
+from rpython.translator.revdb.test.test_basic import BaseRecordingTests
+from rpython.translator.revdb.test.test_basic import InteractiveTests
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib import rthread
+from rpython.rlib import revdb
+
+from rpython.translator.revdb.message import *
+
+
+_sleep = rffi.llexternal('sleep', [rffi.UINT], rffi.UINT)
+
+
+class TestThreadRecording(BaseRecordingTests):
+
+    def test_thread_simple(self):
+        def bootstrap():
+            rthread.gc_thread_start()
+            _sleep(1)
+            print "BB"
+            _sleep(2)
+            print "BBB"
+            rthread.gc_thread_die()
+
+        def main(argv):
+            print "A"
+            rthread.start_new_thread(bootstrap, ())
+            for i in range(2):
+                _sleep(2)
+                print "AAAA"
+            return 9
+
+        self.compile(main, backendopt=False, thread=True)
+        out = self.run('Xx')
+        # should have printed A, BB, AAAA, BBB, AAAA
+        rdb = self.fetch_rdb([self.exename, 'Xx'])
+        th_A = rdb.main_thread_id
+        rdb.write_call("A\n")
+        rdb.same_stack()      # RPyGilAllocate()
+        rdb.gil_release()
+
+        th_B = rdb.switch_thread()
+        assert th_B != th_A
+        b = rdb.next('!h'); assert 300 <= b < 310   # "callback": start thread
+        rdb.gil_release()
+
+        rdb.switch_thread(th_A)
+        rdb.same_stack()      # start_new_thread returns
+        x = rdb.next(); assert x == th_B    # result is the 'th_B' id
+        rdb.gil_release()
+
+        rdb.switch_thread(th_B)
+        rdb.same_stack()      # sleep()
+        rdb.next('i')         # sleep()
+        rdb.write_call("BB\n")
+        rdb.gil_release()
+
+        rdb.switch_thread(th_A)
+        rdb.same_stack()      # sleep()
+        rdb.next('i')         # sleep()
+        rdb.write_call("AAAA\n")
+        rdb.gil_release()
+
+        rdb.switch_thread(th_B)
+        rdb.same_stack()      # sleep()
+        rdb.next('i')         # sleep()
+        rdb.write_call("BBB\n")
+        rdb.gil_release()
+
+        rdb.switch_thread(th_A)
+        rdb.same_stack()      # sleep()
+        rdb.next('i')         # sleep()
+        rdb.write_call("AAAA\n")
+        rdb.done()
+
+
+class TestThreadInteractive(InteractiveTests):
+    expected_stop_points = 5
+
+    def setup_class(cls):
+        from rpython.translator.revdb.test.test_basic import compile, run
+        def bootstrap():
+            rthread.gc_thread_start()
+            _sleep(1)
+            revdb.stop_point()
+            _sleep(2)
+            revdb.stop_point()
+            rthread.gc_thread_die()
+
+        def main(argv):
+            revdb.stop_point()
+            rthread.start_new_thread(bootstrap, ())
+            for i in range(2):
+                _sleep(2)
+                revdb.stop_point()
+            print "ok"
+            return 9
+
+        compile(cls, main, backendopt=False, thread=True)
+        assert run(cls, '') == 'ok\n'
+
+    def test_go(self):
+        child = self.replay()
+        for i in range(2, 6):
+            child.send(Message(CMD_FORWARD, 1))
+            child.expect(ANSWER_READY, i, Ellipsis)
+        child.send(Message(CMD_FORWARD, 1))
+        child.expect(ANSWER_AT_END)

From pypy.commits at gmail.com  Thu Aug 11 04:02:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 01:02:56 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fix test
Message-ID: <57ac3130.4219c20a.21e8e.69fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86135:6a05c5fb0a03
Date: 2016-08-11 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/6a05c5fb0a03/

Log:	fix test

diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py
--- a/rpython/translator/revdb/test/test_weak.py
+++ b/rpython/translator/revdb/test/test_weak.py
@@ -203,6 +203,7 @@
                 assert time == i + 1
                 y = intmask(rdb.next('q')); assert y == -1
                 triggered = True
+            rdb.gil_release()
             rdb.same_stack()
             j = rdb.next()
             assert j == i + 1000000 * triggered
@@ -215,6 +216,7 @@
                     assert uid > 0 and uid not in uid_seen
                     uid_seen.add(uid)
                     lst.append(uid)
+                rdb.gil_release()
                 rdb.same_stack()
                 totals.append((lst, intmask(rdb.next())))
         x = rdb.next('q'); assert x == 3000    # number of stop points

From pypy.commits at gmail.com  Thu Aug 11 04:06:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 01:06:22 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: trying
Message-ID: <57ac31fe.031dc20a.de9d1.6eff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86136:0140e350104d
Date: 2016-08-11 10:05 +0200
http://bitbucket.org/pypy/pypy/changeset/0140e350104d/

Log:	trying

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -56,7 +56,7 @@
 ])
 
 reverse_debugger_disable_modules = set([
-    "thread", "_continuation", "_vmprof", "_multiprocessing",
+    "_continuation", "_vmprof", "_multiprocessing",
     ])
 
 # XXX this should move somewhere else, maybe to platform ("is this posixish"

From pypy.commits at gmail.com  Thu Aug 11 04:46:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 01:46:20 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Careful,
 we need the gil_acquire byte as well: otherwise effects
Message-ID: <57ac3b5c.09afc20a.90904.7b91@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86137:8ac394503ebe
Date: 2016-08-11 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/8ac394503ebe/

Log:	Careful, we need the gil_acquire byte as well: otherwise effects
	like writing to GC objects might occur at the wrong time

diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py
--- a/rpython/translator/revdb/gencsupp.py
+++ b/rpython/translator/revdb/gencsupp.py
@@ -83,18 +83,19 @@
         return call_code   # a hack for ll_call_destructor() to mean
                            # that the calls should really be done
     #
-    # hack: we don't need the flag for at least this common function
-    if call_code == 'RPyGilAcquire();':
-        return 'RPY_REVDB_CALL_GILCTRL(%s);' % (call_code,)
-    if call_code == 'RPyGilRelease();':
+    if call_code in ('RPyGilAcquire();', 'RPyGilRelease();'):
         # Could also work with a regular RPY_REVDB_CALL_VOID, but we
-        # use a different byte (0xFD instead of 0xFC) to detect more
-        # sync misses.  In a single-threaded environment this 0xFD
+        # use a different byte (0xFD, 0xFE instead of 0xFC) to detect more
+        # sync misses.  In a single-threaded environment this 0xFD or 0xFE
         # byte is not needed at all, but in a multi-threaded
         # environment it ensures that during replaying, just after
-        # reading the 0xFD, we switch to a different thread if needed
+        # reading the 0xFD or 0xFE, we switch to a different thread if needed
         # (actually implemented with stacklets).
-        return 'RPY_REVDB_CALL_GIL(%s);' % (call_code,)
+        if call_code == 'RPyGilAcquire();':
+            byte = '0xFD'
+        else:
+            byte = '0xFE'
+        return 'RPY_REVDB_CALL_GIL(%s, %s);' % (call_code, byte)
     #
     tp = funcgen.lltypename(v_result)
     if tp == 'void @':
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -57,8 +57,8 @@
 static char rpy_rev_buffer[16384];    /* max. 32768 */
 int rpy_rev_fileno = -1;
 static char flag_io_disabled = FID_REGULAR_MODE;
-__thread bool_t rpy_active_thread;
-static bool_t *rpy_active_thread_ptr;
+__thread int rpy_active_thread;
+static int *rpy_active_thread_ptr;
 
 
 static void setup_record_mode(int argc, char *argv[]);
@@ -1680,7 +1680,8 @@
 RPY_EXTERN
 void rpy_reverse_db_bad_acquire_gil(void)
 {
-    fprintf(stderr, "out of sync: unexpected byte in log (at acquire_gil)\n");
+    fprintf(stderr, "out of sync: unexpected byte in log "
+                    " (at acquire_gil or release_gil)\n");
     exit(1);
 }
 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -15,7 +15,7 @@
 # error "explicit RPY_RDB_REPLAY: not really supported"
 #endif
     bool_t watch_enabled;
-    long lock;
+    int lock;
     char *buf_p, *buf_limit, *buf_readend;
     uint64_t stop_point_seen, stop_point_break;
     uint64_t unique_id_seen, unique_id_break;
@@ -23,7 +23,7 @@
 
 RPY_EXTERN rpy_revdb_t rpy_revdb;
 RPY_EXTERN int rpy_rev_fileno;
-RPY_EXTERN __thread bool_t rpy_active_thread;
+RPY_EXTERN __thread int rpy_active_thread;
 
 
 /* ------------------------------------------------------------ */
@@ -66,11 +66,15 @@
 /* Acquire/release the lock around EMIT_RECORD, because it may be
    called without holding the GIL.  Note that we're always
    single-threaded during replaying: the lock is only useful during
-   recording. */
+   recording.  
+
+   Implementation trick: use 'a >= b' to mean 'a || !b' (the two
+   variables can only take the values 0 or 1).
+*/
 #define _RPY_REVDB_LOCK()                                               \
     {                                                                   \
-        bool_t _lock_contention = pypy_lock_test_and_set(&rpy_revdb.lock, 1); \
-        if (_lock_contention || !rpy_active_thread)                     \
+        int _lock_contention = pypy_lock_test_and_set(&rpy_revdb.lock, 1); \
+        if (_lock_contention >= rpy_active_thread)                      \
             rpy_reverse_db_lock_acquire(_lock_contention);              \
     }
 #define _RPY_REVDB_UNLOCK()                                             \
@@ -143,17 +147,17 @@
             rpy_reverse_db_invoke_callback(_re);                        \
     }
 
-#define RPY_REVDB_CALL_GIL(call_code)                                   \
+#define RPY_REVDB_CALL_GIL(call_code, byte)                             \
     if (!RPY_RDB_REPLAY) {                                              \
         call_code                                                       \
         _RPY_REVDB_LOCK();                                              \
-        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, 0xFD)                \
+        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, byte)                \
         _RPY_REVDB_UNLOCK();                                            \
     }                                                                   \
     else {                                                              \
         unsigned char _re;                                              \
         _RPY_REVDB_EMIT_REPLAY(unsigned char _e, _re)                   \
-        if (_re != 0xFD)                                                \
+        if (_re != byte)                                                \
             rpy_reverse_db_bad_acquire_gil();                           \
     }
 
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -95,12 +95,16 @@
         x = self.next(); assert x == len(expected_string)
         self.same_stack()   # errno
         x = self.next('i'); assert x == 0      # errno
+        self.gil_acquire()
 
     def same_stack(self):
         x = self.next('c'); assert x == '\xFC'
 
+    def gil_acquire(self):
+        x = self.next('c'); assert x == '\xFD'
+
     def gil_release(self):
-        x = self.next('c'); assert x == '\xFD'
+        x = self.next('c'); assert x == '\xFE'
 
     def switch_thread(self, expected=None):
         th, = self.special_packet(ASYNC_THREAD_SWITCH, 'q')
diff --git a/rpython/translator/revdb/test/test_callback.py b/rpython/translator/revdb/test/test_callback.py
--- a/rpython/translator/revdb/test/test_callback.py
+++ b/rpython/translator/revdb/test/test_callback.py
@@ -66,16 +66,20 @@
         rdb.gil_release()
         rdb.same_stack()                        # callmesimple()
         x = rdb.next('i'); assert x == 55555
+        rdb.gil_acquire()
         rdb.write_call('55555\n')
         rdb.gil_release()
         b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
         x = rdb.next('i'); assert x == 40       # arg n
+        rdb.gil_acquire()
         rdb.gil_release()
         x = rdb.next('!h'); assert x == b       # -> callback
         x = rdb.next('i'); assert x == 3        # arg n
+        rdb.gil_acquire()
         rdb.gil_release()
         rdb.same_stack()                        # <- return in main thread
         x = rdb.next('i'); assert x == 4000 * 300   # return from callme()
+        rdb.gil_acquire()
         rdb.write_call('%s\n' % (4000 * 300,))
         x = rdb.next('q'); assert x == 0      # number of stop points
         assert rdb.done()
@@ -88,14 +92,17 @@
         rdb.gil_release()
         b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
         x = rdb.next('i'); assert x == 40       # arg n
+        rdb.gil_acquire()
         rdb.write_call('40\n')
         rdb.gil_release()
         x = rdb.next('!h'); assert x == b       # -> callback again
         x = rdb.next('i'); assert x == 3        # arg n
+        rdb.gil_acquire()
         rdb.write_call('3\n')
         rdb.gil_release()
         rdb.same_stack()                        # -> return in main thread
         x = rdb.next('i'); assert x == 120      # <- return from callme()
+        rdb.gil_acquire()
         rdb.write_call('120\n')
         x = rdb.next('q'); assert x == 2        # number of stop points
         assert rdb.done()
diff --git a/rpython/translator/revdb/test/test_thread.py b/rpython/translator/revdb/test/test_thread.py
--- a/rpython/translator/revdb/test/test_thread.py
+++ b/rpython/translator/revdb/test/test_thread.py
@@ -41,34 +41,40 @@
         th_B = rdb.switch_thread()
         assert th_B != th_A
         b = rdb.next('!h'); assert 300 <= b < 310   # "callback": start thread
+        rdb.gil_acquire()
         rdb.gil_release()
 
         rdb.switch_thread(th_A)
         rdb.same_stack()      # start_new_thread returns
         x = rdb.next(); assert x == th_B    # result is the 'th_B' id
+        rdb.gil_acquire()
         rdb.gil_release()
 
         rdb.switch_thread(th_B)
-        rdb.same_stack()      # sleep()
+        rdb.same_stack()      # sleep() (finishes here)
         rdb.next('i')         # sleep()
+        rdb.gil_acquire()
         rdb.write_call("BB\n")
         rdb.gil_release()
 
         rdb.switch_thread(th_A)
         rdb.same_stack()      # sleep()
         rdb.next('i')         # sleep()
+        rdb.gil_acquire()
         rdb.write_call("AAAA\n")
         rdb.gil_release()
 
         rdb.switch_thread(th_B)
         rdb.same_stack()      # sleep()
         rdb.next('i')         # sleep()
+        rdb.gil_acquire()
         rdb.write_call("BBB\n")
         rdb.gil_release()
 
         rdb.switch_thread(th_A)
         rdb.same_stack()      # sleep()
         rdb.next('i')         # sleep()
+        rdb.gil_acquire()
         rdb.write_call("AAAA\n")
         rdb.done()
 
diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py
--- a/rpython/translator/revdb/test/test_weak.py
+++ b/rpython/translator/revdb/test/test_weak.py
@@ -204,8 +204,9 @@
                 y = intmask(rdb.next('q')); assert y == -1
                 triggered = True
             rdb.gil_release()
-            rdb.same_stack()
-            j = rdb.next()
+            rdb.same_stack()  #
+            j = rdb.next()    # call to foobar()
+            rdb.gil_acquire()
             assert j == i + 1000000 * triggered
             if triggered:
                 lst = []
@@ -217,8 +218,9 @@
                     uid_seen.add(uid)
                     lst.append(uid)
                 rdb.gil_release()
-                rdb.same_stack()
-                totals.append((lst, intmask(rdb.next())))
+                rdb.same_stack()                           #
+                totals.append((lst, intmask(rdb.next())))  # call to foobar()
+                rdb.gil_acquire()
         x = rdb.next('q'); assert x == 3000    # number of stop points
         #
         assert 1500 <= len(uid_seen) <= 3000

From pypy.commits at gmail.com  Thu Aug 11 06:45:21 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 11 Aug 2016 03:45:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: several changes for translation
Message-ID: <57ac5741.c15e1c0a.1a675.7043@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async
Changeset: r86138:3c775a43b9af
Date: 2016-08-11 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/3c775a43b9af/

Log:	several changes for translation renamed argument name in
	Python.asdl, will resolve an issue for the new type singleton which
	is an object correctly emitting from/to_object for singleton and
	bytes type check for generator added commented old validation code
	for starargs varargannotation ...

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -16,7 +16,7 @@
     if not (space.isinstance_w(w_obj, space.w_str) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
-                   "AST string must be of type str or unicode")
+                    "AST string must be of type str or unicode")
     return w_obj
 
 def get_field(space, w_node, name, optional):
@@ -2568,7 +2568,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Bytes)
-        w_s = self.s.to_object(space)  # bytes
+        w_s = self.s  # bytes
         space.setattr(w_node, space.wrap('s'), w_s)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -2581,7 +2581,7 @@
         w_s = get_field(space, w_node, 's', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _s = bytes.from_object(space, w_s)
+        _s = check_string(space, w_s)
         if _s is None:
             raise_required_value(space, w_node, 's')
         _lineno = space.int_w(w_lineno)
@@ -2593,8 +2593,8 @@
 
 class NameConstant(expr):
 
-    def __init__(self, value, lineno, col_offset):
-        self.value = value
+    def __init__(self, single, lineno, col_offset):
+        self.single = single
         expr.__init__(self, lineno, col_offset)
 
     def walkabout(self, visitor):
@@ -2605,8 +2605,8 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_NameConstant)
-        w_value = self.value.to_object(space)  # singleton
-        space.setattr(w_node, space.wrap('value'), w_value)
+        w_single = self.single  # singleton
+        space.setattr(w_node, space.wrap('single'), w_single)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
         w_col_offset = space.wrap(self.col_offset)  # int
@@ -2615,17 +2615,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        w_value = get_field(space, w_node, 'value', False)
+        w_single = get_field(space, w_node, 'single', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _value = singleton.from_object(space, w_value)
-        if _value is None:
-            raise_required_value(space, w_node, 'value')
+        _single = w_single
+        if _single is None:
+            raise_required_value(space, w_node, 'single')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
-        return NameConstant(_value, _lineno, _col_offset)
-
-State.ast_type('NameConstant', 'expr', ['value'])
+        return NameConstant(_single, _lineno, _col_offset)
+
+State.ast_type('NameConstant', 'expr', ['single'])
 
 
 class Ellipsis(expr):
@@ -2952,8 +2952,8 @@
 
 class Const(expr):
 
-    def __init__(self, value, lineno, col_offset):
-        self.value = value
+    def __init__(self, obj, lineno, col_offset):
+        self.obj = obj
         expr.__init__(self, lineno, col_offset)
 
     def walkabout(self, visitor):
@@ -2964,8 +2964,8 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Const)
-        w_value = self.value  # object
-        space.setattr(w_node, space.wrap('value'), w_value)
+        w_obj = self.obj  # object
+        space.setattr(w_node, space.wrap('obj'), w_obj)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
         w_col_offset = space.wrap(self.col_offset)  # int
@@ -2974,17 +2974,17 @@
 
     @staticmethod
     def from_object(space, w_node):
-        w_value = get_field(space, w_node, 'value', False)
+        w_obj = get_field(space, w_node, 'obj', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _value = w_value
-        if _value is None:
-            raise_required_value(space, w_node, 'value')
+        _obj = w_obj
+        if _obj is None:
+            raise_required_value(space, w_node, 'obj')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
-        return Const(_value, _lineno, _col_offset)
-
-State.ast_type('Const', 'expr', ['value'])
+        return Const(_obj, _lineno, _col_offset)
+
+State.ast_type('Const', 'expr', ['obj'])
 
 
 class expr_context(AST):
diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py
--- a/pypy/interpreter/astcompiler/asthelpers.py
+++ b/pypy/interpreter/astcompiler/asthelpers.py
@@ -152,7 +152,7 @@
 
     def as_node_list(self, space):
         try:
-            values_w = space.unpackiterable(self.value)
+            values_w = space.unpackiterable(self.obj)
         except OperationError:
             return None
         line = self.lineno
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1047,7 +1047,7 @@
 
     def visit_Const(self, const):
         self.update_position(const.lineno)
-        self.load_const(const.value)
+        self.load_const(const.obj)
 
     def visit_Ellipsis(self, e):
         self.load_const(self.space.w_Ellipsis)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -65,7 +65,7 @@
 class __extend__(ast.Const):
 
     def as_constant(self):
-        return self.value
+        return self.obj
 
 class __extend__(ast.Index):
     def as_constant(self):
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -71,7 +71,8 @@
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
          | Bytes(bytes s)
-         | NameConstant(singleton value)
+         -- PyPy mod. first argument name must not be value
+         | NameConstant(singleton single)
          | Ellipsis
 
          -- the following expression can appear in assignment context
@@ -83,7 +84,7 @@
          | Tuple(expr* elts, expr_context ctx)
 
          -- PyPy modification
-         | Const(object value)
+         | Const(object obj)
 
           -- col_offset is the byte offset in the utf8 string the parser uses
           attributes (int lineno, int col_offset)
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -130,7 +130,7 @@
             if field.opt:
                 wrapper += " if %s is not None else space.w_None" % (value,)
             return wrapper
-        elif field.type in ("object", "string"):
+        elif field.type in ("object", "singleton", "string", "bytes"):
             return value
         elif field.type in ("int", "bool"):
             return "space.wrap(%s)" % (value,)
@@ -145,9 +145,9 @@
     def get_value_extractor(self, field, value):
         if field.type in self.data.simple_types:
             return "%s.from_object(space, %s)" % (field.type, value)
-        elif field.type in ("object",):
+        elif field.type in ("object","singleton"):
             return value
-        elif field.type in ("string",):
+        elif field.type in ("string","bytes"):
             return "check_string(space, %s)" % (value,)
         elif field.type in ("identifier",):
             if field.opt:
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -152,15 +152,15 @@
 
     def visit_arguments(self, node):
         self.visit_sequence(node.args)
-        if node.varargannotation:
-            if not node.vararg:
-                raise ValidationError("varargannotation but no vararg on arguments")
-            self._validate_expr(node.varargannotation)
+        # XXX py3.5 missing if node.varargannotation:
+        # XXX py3.5 missing     if not node.vararg:
+        # XXX py3.5 missing         raise ValidationError("varargannotation but no vararg on arguments")
+        # XXX py3.5 missing     self._validate_expr(node.varargannotation)
         self.visit_sequence(node.kwonlyargs)
-        if node.kwargannotation:
-            if not node.kwarg:
-                raise ValidationError("kwargannotation but no kwarg on arguments")
-            self._validate_expr(node.kwargannotation)
+        # XXX py3.5 missing if node.kwargannotation:
+        # XXX py3.5 missing     if not node.kwarg:
+        # XXX py3.5 missing         raise ValidationError("kwargannotation but no kwarg on arguments")
+        # XXX py3.5 missing     self._validate_expr(node.kwargannotation)
         if self._len(node.defaults) > self._len(node.args):
             raise ValidationError("more positional defaults than args on arguments")
         if self._len(node.kw_defaults) != self._len(node.kwonlyargs):
@@ -184,10 +184,10 @@
         self._validate_exprs(node.bases)
         self.visit_sequence(node.keywords)
         self._validate_exprs(node.decorator_list)
-        if node.starargs:
-            self._validate_expr(node.starargs)
-        if node.kwargs:
-            self._validate_expr(node.kwargs)
+        # XXX py3.5 missing if node.starargs:
+        # XXX py3.5 missing   self._validate_expr(node.starargs)
+        # XXX py3.5 missing if node.kwargs:
+        # XXX py3.5 missing     self._validate_expr(node.kwargs)
 
     def visit_Return(self, node):
         if node.value:
@@ -373,10 +373,10 @@
         self._validate_expr(node.func)
         self._validate_exprs(node.args)
         self.visit_sequence(node.keywords)
-        if node.starargs:
-            self._validate_expr(node.starargs)
-        if node.kwargs:
-            self._validate_expr(node.kwargs)
+        # XXX py3.5 missing if node.starargs:
+        # XXX py3.5 missing     self._validate_expr(node.starargs)
+        # XXX py3.5 missing if node.kwargs:
+        # XXX py3.5 missing     self._validate_expr(node.kwargs)
 
     def visit_Num(self, node):
         space = self.space
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -316,7 +316,8 @@
         res = space.get_and_call_function(w_await, self)
         if res is not None:
             if (isinstance(res, Coroutine) or
-                res.pycode.co_flags & consts.CO_ITERABLE_COROUTINE):
+                (isinstance(res, GeneratorIterator) and \
+                 res.pycode.co_flags & consts.CO_ITERABLE_COROUTINE)):
                 raise oefmt(space.w_TypeError,
                             "__await__() returned a coroutine")
             elif space.lookup(self, "__next__") is None:

From pypy.commits at gmail.com  Thu Aug 11 10:15:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 07:15:18 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Restore usage of
 RDBSignalActionFlag, which might have been removed by
Message-ID: <57ac8876.6aaac20a.437bd.03f9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86139:260de9d411ea
Date: 2016-08-11 14:01 +0200
http://bitbucket.org/pypy/pypy/changeset/260de9d411ea/

Log:	Restore usage of RDBSignalActionFlag, which might have been removed
	by accident

diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py
--- a/pypy/module/signal/__init__.py
+++ b/pypy/module/signal/__init__.py
@@ -46,7 +46,10 @@
         space.check_signal_action = interp_signal.CheckSignalAction(space)
         space.actionflag.register_periodic_action(space.check_signal_action,
                                                   use_bytecode_counter=False)
-        if not space.config.translation.reverse_debugger:
+        if space.config.translation.reverse_debugger:
+            from pypy.interpreter.reverse_debugging import RDBSignalActionFlag
+            space.actionflag.__class__ = RDBSignalActionFlag
+        else:
             space.actionflag.__class__ = interp_signal.SignalActionFlag
         # xxx yes I know the previous line is a hack
 

From pypy.commits at gmail.com  Thu Aug 11 10:15:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 07:15:20 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Translation fixes
Message-ID: <57ac8878.2472c20a.3b6d9.05ff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86140:c4ffe110cd17
Date: 2016-08-11 14:01 +0200
http://bitbucket.org/pypy/pypy/changeset/c4ffe110cd17/

Log:	Translation fixes

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -131,7 +131,8 @@
     _RPY_REVDB_UNLOCK();
 }
 
-static void reverse_db_teardown(void)
+RPY_EXTERN
+void rpy_reverse_db_teardown(void)
 {
     uint64_t stop_points;
     if (!RPY_RDB_REPLAY) {
@@ -142,8 +143,8 @@
            rpy_reverse_db_fetch(), which has nothing more to fetch now */
         rpy_revdb.buf_limit += 1;
     }
-    _RPY_REVDB_EMIT_L(stop_points = rpy_revdb.stop_point_seen; ,
-                      uint64_t _e, stop_points, /*must_lock=*/0);
+    _RPY_REVDB_EMIT_L(stop_points = rpy_revdb.stop_point_seen;,
+                      uint64_t _e, stop_points);
 
     if (!RPY_RDB_REPLAY) {
         rpy_reverse_db_flush();
@@ -523,8 +524,7 @@
         else
             r->re_off_prev = 1;    /* any number > 0 */
 
-        _RPY_REVDB_EMIT_L(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive,
-                          /*must_lock=*/0);
+        _RPY_REVDB_EMIT_L(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive);
 
         if (!RPY_RDB_REPLAY) {
             _RPY_REVDB_UNLOCK();
@@ -571,8 +571,7 @@
                                                   WEAKREF_AFTERWARDS_ALIVE);
                 r->re_off_prev = recording_offset();
             }
-            _RPY_REVDB_EMIT_L(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive,
-                              /*must_lock=*/0);
+            _RPY_REVDB_EMIT_L(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive);
 
             if (!RPY_RDB_REPLAY) {
                 _RPY_REVDB_UNLOCK();
@@ -684,7 +683,7 @@
     m->entry_point(m->argc, m->argv);
 
     /* main thread finished, program stops */
-    reverse_db_teardown();
+    rpy_reverse_db_teardown();
 
     /* unreachable */
     abort();
@@ -730,7 +729,7 @@
 {
     if (!RPY_RDB_REPLAY) {
         int exitcode = (int)entry_point(argc, argv);
-        reverse_db_teardown();
+        rpy_reverse_db_teardown();
         return exitcode;
     }
     else {
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -103,17 +103,21 @@
             variable = _e;                                              \
         }
 
-#define _RPY_REVDB_EMIT_L(normal_code, decl_e, variable, must_lock)     \
+#define _RPY_REVDB_EMIT_L(normal_code, decl_e, variable)                \
     if (!RPY_RDB_REPLAY) {                                              \
         normal_code                                                     \
-        if (must_lock) _RPY_REVDB_LOCK();                               \
         _RPY_REVDB_EMIT_RECORD_L(decl_e, variable)                      \
-        if (must_lock) _RPY_REVDB_UNLOCK();                             \
     } else                                                              \
         _RPY_REVDB_EMIT_REPLAY(decl_e, variable)
 
 #define RPY_REVDB_EMIT(normal_code, decl_e, variable)                   \
-    _RPY_REVDB_EMIT_L(normal_code, decl_e, variable, 1)
+    if (!RPY_RDB_REPLAY) {                                              \
+        normal_code                                                     \
+        _RPY_REVDB_LOCK();                                              \
+        _RPY_REVDB_EMIT_RECORD_L(decl_e, variable)                      \
+        _RPY_REVDB_UNLOCK();                                            \
+    } else                                                              \
+        _RPY_REVDB_EMIT_REPLAY(decl_e, variable)
 
 #define RPY_REVDB_EMIT_VOID(normal_code)                                \
     if (!RPY_RDB_REPLAY) { normal_code } else { }

From pypy.commits at gmail.com  Thu Aug 11 10:18:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 11 Aug 2016 07:18:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: merged py3k changes,
 generatorentry_driver was used twice, that is now allowed
Message-ID: <57ac894e.8f8e1c0a.8df7f.1f77@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async
Changeset: r86141:ce8bb88f9458
Date: 2016-08-11 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/ce8bb88f9458/

Log:	merged py3k changes, generatorentry_driver was used twice, that is
	now allowed

diff --git a/lib-python/3/test/test_hash.py b/lib-python/3/test/test_hash.py
--- a/lib-python/3/test/test_hash.py
+++ b/lib-python/3/test/test_hash.py
@@ -198,7 +198,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
     repr_ = None
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -326,6 +326,12 @@
                         "of type '%T'", res)
         return res
 
+def get_printable_coroutine_location_genentry(bytecode):
+    return '%s <coroutine>' % (bytecode.get_repr(),)
+coroutineentry_driver = jit.JitDriver(greens=['pycode'],
+                                      reds=['gen', 'w_arg', 'operr'],
+                                      get_printable_location = get_printable_coroutine_location_genentry,
+                                      name='coroutineentry')
 
 class Coroutine(W_Root):
     "A coroutine object."
@@ -511,8 +517,8 @@
         pycode = self.pycode
         if pycode is not None:
             if jit.we_are_jitted() and should_not_inline(pycode):
-                generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
-                                                    operr=operr, pycode=pycode)
+                coroutineentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
+                                                      operr=operr, pycode=pycode)
         return self._send_ex(w_arg, operr)
 
     def _send_ex(self, w_arg, operr):
diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -175,7 +175,7 @@
          "__new__"      : make_new(class_name),
          }
     pycppclass = metacpp(class_name, _drop_cycles(bases), d)
- 
+
     # cache result early so that the class methods can find the class itself
     setattr(scope, final_class_name, pycppclass)
 
@@ -192,13 +192,10 @@
     for dm_name in cppclass.get_datamember_names():
         cppdm = cppclass.get_datamember(dm_name)
 
-        # here, setattr() can not be used, because a data member can shadow one in
-        # its base class, resulting in the __set__() of its base class being called
-        # by setattr(); so, store directly on the dictionary
-        pycppclass.__dict__[dm_name] = cppdm
+        setattr(pycppclass, dm_name, cppdm)
         import cppyy
         if cppyy._is_static(cppdm):     # TODO: make this a method of cppdm
-            metacpp.__dict__[dm_name] = cppdm
+            setattr(metacpp, dm_name, cppdm)
 
     # the call to register will add back-end specific pythonizations and thus
     # needs to run first, so that the generic pythonizations can use them
@@ -413,7 +410,7 @@
         lib = cppyy._load_dictionary(name)
         _loaded_dictionaries[name] = lib
         return lib
-    
+
 def _init_pythonify():
     # cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause cppyy to be loaded
diff --git a/pypy/module/cpyext/dictproxyobject.py b/pypy/module/cpyext/dictproxyobject.py
--- a/pypy/module/cpyext/dictproxyobject.py
+++ b/pypy/module/cpyext/dictproxyobject.py
@@ -1,67 +1,7 @@
-# Read-only proxy for mappings. PyPy does not have a separate type for
-# type.__dict__, so PyDictProxy_New has to use a custom read-only mapping.
-
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.interpreter.typedef import TypeDef, interp2app
+from pypy.objspace.std.dictproxyobject import W_DictProxyObject
 from pypy.module.cpyext.api import cpython_api, build_type_checkers
 from pypy.module.cpyext.pyobject import PyObject
 
-class W_DictProxyObject(W_Root):
-    "Read-only proxy for mappings."
-
-    def __init__(self, w_mapping):
-        self.w_mapping = w_mapping
-
-    def descr_len(self, space):
-        return space.len(self.w_mapping)
-
-    def descr_getitem(self, space, w_key):
-        return space.getitem(self.w_mapping, w_key)
-
-    def descr_contains(self, space, w_key):
-        return space.contains(self.w_mapping, w_key)
-
-    def descr_iter(self, space):
-        return space.iter(self.w_mapping)
-
-    def descr_str(self, space):
-        return space.str(self.w_mapping)
-
-    def descr_repr(self, space):
-        return space.repr(self.w_mapping)
-
-    @unwrap_spec(w_default=WrappedDefault(None))
-    def get_w(self, space, w_key, w_default):
-        return space.call_method(self.w_mapping, "get", w_key, w_default)
-
-    def keys_w(self, space):
-        return space.call_method(self.w_mapping, "keys")
-
-    def values_w(self, space):
-        return space.call_method(self.w_mapping, "values")
-
-    def items_w(self, space):
-        return space.call_method(self.w_mapping, "items")
-
-    def copy_w(self, space):
-        return space.call_method(self.w_mapping, "copy")
-
-W_DictProxyObject.typedef = TypeDef(
-    'mappingproxy',
-    __len__=interp2app(W_DictProxyObject.descr_len),
-    __getitem__=interp2app(W_DictProxyObject.descr_getitem),
-    __contains__=interp2app(W_DictProxyObject.descr_contains),
-    __iter__=interp2app(W_DictProxyObject.descr_iter),
-    __str__=interp2app(W_DictProxyObject.descr_str),
-    __repr__=interp2app(W_DictProxyObject.descr_repr),
-    get=interp2app(W_DictProxyObject.get_w),
-    keys=interp2app(W_DictProxyObject.keys_w),
-    values=interp2app(W_DictProxyObject.values_w),
-    items=interp2app(W_DictProxyObject.items_w),
-    copy=interp2app(W_DictProxyObject.copy_w)
-)
-
 PyDictProxy_Check, PyDictProxy_CheckExact = build_type_checkers(
     "DictProxy", W_DictProxyObject)
 
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -417,8 +417,7 @@
             init = """PyObject *mod = PyModule_Create(&moduledef);"""
             if more_init:
                 init += more_init
-            else:
-                init += "\nreturn mod;"
+            init += "\nreturn mod;"
             return import_module(space, name=modname, init=init, body=body,
                                  w_include_dirs=w_include_dirs,
                                  PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
diff --git a/pypy/module/cpyext/test/test_import_module.c b/pypy/module/cpyext/test/test_import_module.c
--- a/pypy/module/cpyext/test/test_import_module.c
+++ b/pypy/module/cpyext/test/test_import_module.c
@@ -1,17 +1,20 @@
 #include "Python.h"
 /* Initialize this module. */
 
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "test_import_module",
+    NULL,
+    -1,
+    NULL, NULL, NULL, NULL, NULL
+};
+
 PyMODINIT_FUNC
-inittest_import_module(void)
+PyInit_test_import_module(void)
 {
-	PyObject *m, *d;
-
-	m = Py_InitModule("test_import_module", NULL);
-	if (m == NULL)
-	    return;
-	d = PyModule_GetDict(m);
-	if (d) {
-        PyDict_SetItemString(d, "TEST", (PyObject *) Py_None);
-    }
-   	/* No need to check the error here, the caller will do that */
+    PyObject* m = PyModule_Create(&moduledef);
+    if (m == NULL)
+	    return NULL;
+    PyModule_AddObject(m, "TEST", (PyObject *) Py_None);
+    return m;
 }
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py
--- a/pypy/module/cpyext/test/test_number.py
+++ b/pypy/module/cpyext/test/test_number.py
@@ -1,5 +1,6 @@
 from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 class TestIterator(BaseApiTest):
     def test_check(self, space, api):
@@ -63,7 +64,9 @@
         assert 9 == space.unwrap(
             api.PyNumber_InPlacePower(space.wrap(3), space.wrap(2), space.w_None))
 
-    def test_PyNumber_Check(self):        
+
+class AppTestCNumber(AppTestCpythonExtensionBase):
+    def test_PyNumber_Check(self):
         mod = self.import_extension('foo', [
             ("test_PyNumber_Check", "METH_VARARGS",
              '''
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -277,9 +277,41 @@
                      args->ob_type->tp_dict, "copy");
                  Py_INCREF(method);
                  return method;
-             ''')])
+             '''),
+            ("get_type_dict", "METH_O",
+             '''
+                PyObject* value = args->ob_type->tp_dict;
+                if (value == NULL) value = Py_None;
+                Py_INCREF(value);
+                return value;
+             '''),
+            ])
         obj = foo.new()
         assert module.read_tp_dict(obj) == foo.fooType.copy
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
+        class A(object):
+            pass
+        obj = A()
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
+        d = module.get_type_dict(1)
+        assert type(d) is dict
+        try:
+            d["_some_attribute"] = 1
+        except TypeError:  # on PyPy, int.__dict__ is really immutable
+            pass
+        else:
+            assert int._some_attribute == 1
+            del d["_some_attribute"]
 
     def test_custom_allocation(self):
         foo = self.import_module("foo")
@@ -348,6 +380,21 @@
 
         api.Py_DecRef(ref)
 
+    def test_type_dict(self, space, api):
+        w_class = space.appexec([], """():
+            class A(object):
+                pass
+            return A
+            """)
+        ref = make_ref(space, w_class)
+
+        py_type = rffi.cast(PyTypeObjectPtr, ref)
+        w_dict = from_ref(space, py_type.c_tp_dict)
+        w_name = space.newunicode(u'a')
+        space.setitem(w_dict, w_name, space.wrap(1))
+        assert space.int_w(space.getattr(w_class, w_name)) == 1
+        space.delitem(w_dict, w_name)
+
     def test_multiple_inheritance(self, space, api):
         w_class = space.appexec([], """():
             class A(object):
@@ -779,7 +826,7 @@
             """, more_init="""
             IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT;
             IntLike_Type.tp_as_number = &intlike_as_number;
-            intlike_as_number.nb_bool = intlike_nb_nonzero;
+            intlike_as_number.nb_bool = intlike_nb_bool;
             intlike_as_number.nb_int = intlike_nb_int;
             PyType_Ready(&IntLike_Type);
             """)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -192,7 +192,7 @@
     py_methoddescr.c_d_method = w_obj.ml
 
 def classmethoddescr_realize(space, obj):
-    # XXX NOT TESTED When is this ever called? 
+    # XXX NOT TESTED When is this ever called?
     method = rffi.cast(lltype.Ptr(PyMethodDef), obj)
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
     w_obj = space.allocate_instance(W_PyCClassMethodObject, w_type)
@@ -201,7 +201,7 @@
     return w_obj
 
 def methoddescr_realize(space, obj):
-    # XXX NOT TESTED When is this ever called? 
+    # XXX NOT TESTED When is this ever called?
     method = rffi.cast(lltype.Ptr(PyMethodDef), obj)
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
     w_obj = space.allocate_instance(W_PyCMethodObject, w_type)
@@ -272,12 +272,12 @@
         if len(slot_names) == 1:
             if not getattr(pto, slot_names[0]):
                 setattr(pto, slot_names[0], slot_func_helper)
-        elif (w_type.getname(space) in ('list', 'tuple') and 
+        elif (w_type.getname(space) in ('list', 'tuple') and
               slot_names[0] == 'c_tp_as_number'):
             # XXX hack - hwo can we generalize this? The problem is method
             # names like __mul__ map to more than one slot, and we have no
             # convenient way to indicate which slots CPython have filled
-            # 
+            #
             # We need at least this special case since Numpy checks that
             # (list, tuple) do __not__ fill tp_as_number
             pass
@@ -767,8 +767,8 @@
 
     if w_obj.is_cpytype():
         Py_DecRef(space, pto.c_tp_dict)
-        w_dict = w_obj.getdict(space)
-        pto.c_tp_dict = make_ref(space, w_dict)
+    w_dict = w_obj.getdict(space)
+    pto.c_tp_dict = make_ref(space, w_dict)
 
 @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL)
 def PyType_IsSubtype(space, a, b):
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -541,6 +541,8 @@
         t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
         t_ref[0] = tt
         pbuf = c_localtime(t_ref)
+        rffi.setintfield(pbuf, "c_tm_year",
+                         rffi.getintfield(pbuf, "c_tm_year") + 1900)
         lltype.free(t_ref, flavor='raw')
         if not pbuf:
             raise OperationError(space.w_ValueError,
@@ -584,7 +586,7 @@
     if rffi.getintfield(glob_buf, 'c_tm_wday') < -1:
         raise oefmt(space.w_ValueError, "day of week out of range")
 
-    rffi.setintfield(glob_buf, 'c_tm_year', y - 1900)
+    rffi.setintfield(glob_buf, 'c_tm_year', y)
     rffi.setintfield(glob_buf, 'c_tm_mon',
                      rffi.getintfield(glob_buf, 'c_tm_mon') - 1)
     rffi.setintfield(glob_buf, 'c_tm_wday',
@@ -648,7 +650,8 @@
         t_ref[0] = seconds
         p = c_localtime(t_ref)
     if not p:
-        raise oefmt(space.w_ValueError, "unconvertible time")
+        raise oefmt(space.w_OSError, "unconvertible time")
+    rffi.setintfield(p, "c_tm_year", rffi.getintfield(p, "c_tm_year") + 1900)
     return _asctime(space, p)
 
 # by now w_tup is an optional argument (and not *args)
@@ -677,7 +680,7 @@
             w(getif(t_ref, 'c_tm_hour')),
             w(getif(t_ref, 'c_tm_min')),
             w(getif(t_ref, 'c_tm_sec')),
-            w(getif(t_ref, 'c_tm_year') + 1900)]
+            w(getif(t_ref, 'c_tm_year'))]
     return space.mod(w("%.3s %.3s%3d %.2d:%.2d:%.2d %d"),
                      space.newtuple(args))
 
@@ -715,7 +718,7 @@
     lltype.free(t_ref, flavor='raw')
 
     if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
+        raise OperationError(space.w_OSError, space.wrap(_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def mktime(space, w_tup):
@@ -725,6 +728,7 @@
 
     buf = _gettmarg(space, w_tup, allowNone=False)
     rffi.setintfield(buf, "c_tm_wday", -1)
+    rffi.setintfield(buf, "c_tm_year", rffi.getintfield(buf, "c_tm_year") - 1900)
     tt = c_mktime(buf)
     # A return value of -1 does not necessarily mean an error, but tm_wday
     # cannot remain set to -1 if mktime succeeds.
@@ -801,6 +805,8 @@
         rffi.setintfield(buf_value, 'c_tm_isdst', -1)
     elif rffi.getintfield(buf_value, 'c_tm_isdst') > 1:
         rffi.setintfield(buf_value, 'c_tm_isdst', 1)
+    rffi.setintfield(buf_value, "c_tm_year",
+                     rffi.getintfield(buf_value, "c_tm_year") - 1900)
 
     if _WIN:
         # check that the format string contains only valid directives
diff --git a/pypy/objspace/std/classdict.py b/pypy/objspace/std/classdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/classdict.py
@@ -0,0 +1,119 @@
+from rpython.rlib import rerased
+from rpython.rlib.objectmodel import iteritems_with_hash
+
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.objspace.std.dictmultiobject import (
+    DictStrategy, create_iterator_classes)
+from pypy.objspace.std.typeobject import unwrap_cell
+
+
+class ClassDictStrategy(DictStrategy):
+    """Exposes a W_TypeObject.dict_w at app-level.
+
+    Uses getdictvalue() and setdictvalue() to access items.
+    """
+    erase, unerase = rerased.new_erasing_pair("dictproxy")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        w_lookup_type = space.type(w_key)
+        if space.issubtype_w(w_lookup_type, space.w_unicode):
+            return self.getitem_str(w_dict, space.str_w(w_key))
+        else:
+            return None
+
+    def getitem_str(self, w_dict, key):
+        return self.unerase(w_dict.dstorage).getdictvalue(self.space, key)
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if space.is_w(space.type(w_key), space.w_unicode):
+            self.setitem_str(w_dict, self.space.str_w(w_key), w_value)
+        else:
+            raise oefmt(space.w_TypeError,
+                        "cannot add non-string keys to dict of a type")
+
+    def setitem_str(self, w_dict, key, w_value):
+        w_type = self.unerase(w_dict.dstorage)
+        try:
+            w_type.setdictvalue(self.space, key, w_value)
+        except OperationError as e:
+            if not e.match(self.space, self.space.w_TypeError):
+                raise
+            if not w_type.is_cpytype():
+                raise
+            # Allow cpyext to write to type->tp_dict even in the case
+            # of a builtin type.
+            # Like CPython, we assume that this is only done early
+            # after the type is created, and we don't invalidate any
+            # cache.  User code shoud call PyType_Modified().
+            w_type.dict_w[key] = w_value
+
+    def setdefault(self, w_dict, w_key, w_default):
+        w_result = self.getitem(w_dict, w_key)
+        if w_result is not None:
+            return w_result
+        self.setitem(w_dict, w_key, w_default)
+        return w_default
+
+    def delitem(self, w_dict, w_key):
+        space = self.space
+        w_key_type = space.type(w_key)
+        if space.is_w(w_key_type, space.w_unicode):
+            key = self.space.str_w(w_key)
+            if not self.unerase(w_dict.dstorage).deldictvalue(space, key):
+                raise KeyError
+        else:
+            raise KeyError
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage).dict_w)
+
+    def w_keys(self, w_dict):
+        space = self.space
+        w_type = self.unerase(w_dict.dstorage)
+        return space.newlist([_wrapkey(space, key)
+                              for key in w_type.dict_w.iterkeys()])
+
+    def values(self, w_dict):
+        return [unwrap_cell(self.space, w_value) for w_value in
+                self.unerase(w_dict.dstorage).dict_w.itervalues()]
+
+    def items(self, w_dict):
+        space = self.space
+        w_type = self.unerase(w_dict.dstorage)
+        return [space.newtuple([_wrapkey(space, key),
+                                unwrap_cell(space, w_value)])
+                for (key, w_value) in w_type.dict_w.iteritems()]
+
+    def clear(self, w_dict):
+        space = self.space
+        w_type = self.unerase(w_dict.dstorage)
+        if not w_type.is_heaptype():
+            raise oefmt(space.w_TypeError,
+                        "can't clear dictionary of type '%N'", w_type)
+        w_type.dict_w.clear()
+        w_type.mutated(None)
+
+    def getiterkeys(self, w_dict):
+        return self.unerase(w_dict.dstorage).dict_w.iterkeys()
+
+    def getitervalues(self, w_dict):
+        return self.unerase(w_dict.dstorage).dict_w.itervalues()
+
+    def getiteritems_with_hash(self, w_dict):
+        return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w)
+
+    def wrapkey(space, key):
+        return _wrapkey(space, key)
+
+    def wrapvalue(space, value):
+        return unwrap_cell(space, value)
+
+def _wrapkey(space, key):
+    # keys are utf-8 encoded identifiers from type's dict_w
+    return space.wrap(key.decode('utf-8'))
+
+create_iterator_classes(ClassDictStrategy)
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -1,212 +1,95 @@
-from rpython.rlib import rerased
-from rpython.rlib.objectmodel import iteritems_with_hash
+"""
+Read-only proxy for mappings.
 
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.typedef import TypeDef
-from pypy.objspace.std.dictmultiobject import (
-    DictStrategy, W_DictObject, create_iterator_classes)
-from pypy.objspace.std.typeobject import unwrap_cell
+Its main use is as the return type of cls.__dict__.
+"""
 
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
+from pypy.interpreter.typedef import TypeDef, interp2app
 
-class W_DictProxyObject(W_DictObject):
+class W_DictProxyObject(W_Root):
+    "Read-only proxy for mappings."
+
+    def __init__(self, w_mapping):
+        self.w_mapping = w_mapping
+
     @staticmethod
     def descr_new(space, w_type, w_mapping):
         if (not space.lookup(w_mapping, "__getitem__") or
-            space.isinstance_w(w_mapping, space.w_list) or
-            space.isinstance_w(w_mapping, space.w_tuple)):
+                space.isinstance_w(w_mapping, space.w_list) or
+                space.isinstance_w(w_mapping, space.w_tuple)):
             raise oefmt(space.w_TypeError,
-                        "mappingproxy() argument must be a mapping, not %T", w_mapping)
-        strategy = space.fromcache(MappingProxyStrategy)
-        storage = strategy.erase(w_mapping)
-        w_obj = space.allocate_instance(W_DictProxyObject, w_type)
-        W_DictProxyObject.__init__(w_obj, space, strategy, storage)
-        return w_obj
+                        "mappingproxy() argument must be a mapping, not %T",
+                        w_mapping)
+        return W_DictProxyObject(w_mapping)
 
     def descr_init(self, space, __args__):
         pass
 
+    def descr_len(self, space):
+        return space.len(self.w_mapping)
+
+    def descr_getitem(self, space, w_key):
+        return space.getitem(self.w_mapping, w_key)
+
+    def descr_contains(self, space, w_key):
+        return space.contains(self.w_mapping, w_key)
+
+    def descr_iter(self, space):
+        return space.iter(self.w_mapping)
+
+    def descr_str(self, space):
+        return space.str(self.w_mapping)
+
     def descr_repr(self, space):
-        return space.wrap(u"mappingproxy(%s)" % (
-            space.unicode_w(W_DictObject.descr_repr(self, space))))
+        return space.newunicode(u"mappingproxy(%s)" %
+                                (space.unicode_w(space.repr(self.w_mapping)),))
+
+    @unwrap_spec(w_default=WrappedDefault(None))
+    def get_w(self, space, w_key, w_default):
+        return space.call_method(self.w_mapping, "get", w_key, w_default)
+
+    def keys_w(self, space):
+        return space.call_method(self.w_mapping, "keys")
+
+    def values_w(self, space):
+        return space.call_method(self.w_mapping, "values")
+
+    def items_w(self, space):
+        return space.call_method(self.w_mapping, "items")
+
+    def copy_w(self, space):
+        return space.call_method(self.w_mapping, "copy")
+
+cmp_methods = {}
+def make_cmp_method(op):
+    def descr_op(self, space, w_other):
+        return getattr(space, op)(self.w_mapping, w_other)
+    descr_name = 'descr_' + op
+    descr_op.__name__ = descr_name
+    setattr(W_DictProxyObject, descr_name, descr_op)
+    cmp_methods['__%s__' % op] = interp2app(getattr(W_DictProxyObject, descr_name))
+
+for op in ['eq', 'ne', 'gt', 'ge', 'lt', 'le']:
+    make_cmp_method(op)
+
 
 W_DictProxyObject.typedef = TypeDef(
-    "mappingproxy", W_DictObject.typedef,
-    __new__ = interp2app(W_DictProxyObject.descr_new),
-    __init__ = interp2app(W_DictProxyObject.descr_init),
-    __repr__ = interp2app(W_DictProxyObject.descr_repr),
+    'mappingproxy',
+    __new__=interp2app(W_DictProxyObject.descr_new),
+    __init__=interp2app(W_DictProxyObject.descr_init),
+    __len__=interp2app(W_DictProxyObject.descr_len),
+    __getitem__=interp2app(W_DictProxyObject.descr_getitem),
+    __contains__=interp2app(W_DictProxyObject.descr_contains),
+    __iter__=interp2app(W_DictProxyObject.descr_iter),
+    __str__=interp2app(W_DictProxyObject.descr_str),
+    __repr__=interp2app(W_DictProxyObject.descr_repr),
+    get=interp2app(W_DictProxyObject.get_w),
+    keys=interp2app(W_DictProxyObject.keys_w),
+    values=interp2app(W_DictProxyObject.values_w),
+    items=interp2app(W_DictProxyObject.items_w),
+    copy=interp2app(W_DictProxyObject.copy_w),
+    **cmp_methods
 )
-
-
-class DictProxyStrategy(DictStrategy):
-    """Exposes a W_TypeObject.dict_w at app-level.
-
-    Uses getdictvalue() and setdictvalue() to access items.
-    """
-    erase, unerase = rerased.new_erasing_pair("dictproxy")
-    erase = staticmethod(erase)
-    unerase = staticmethod(unerase)
-
-    def getitem(self, w_dict, w_key):
-        space = self.space
-        w_lookup_type = space.type(w_key)
-        if space.issubtype_w(w_lookup_type, space.w_unicode):
-            return self.getitem_str(w_dict, space.str_w(w_key))
-        else:
-            return None
-
-    def getitem_str(self, w_dict, key):
-        return self.unerase(w_dict.dstorage).getdictvalue(self.space, key)
-
-    def setitem(self, w_dict, w_key, w_value):
-        space = self.space
-        if space.is_w(space.type(w_key), space.w_unicode):
-            self.setitem_str(w_dict, self.space.str_w(w_key), w_value)
-        else:
-            raise oefmt(space.w_TypeError,
-                        "cannot add non-string keys to dict of a type")
-
-    def setitem_str(self, w_dict, key, w_value):
-        w_type = self.unerase(w_dict.dstorage)
-        try:
-            w_type.setdictvalue(self.space, key, w_value)
-        except OperationError as e:
-            if not e.match(self.space, self.space.w_TypeError):
-                raise
-            if not w_type.is_cpytype():
-                raise
-            # Allow cpyext to write to type->tp_dict even in the case
-            # of a builtin type.
-            # Like CPython, we assume that this is only done early
-            # after the type is created, and we don't invalidate any
-            # cache.  User code shoud call PyType_Modified().
-            w_type.dict_w[key] = w_value
-
-    def setdefault(self, w_dict, w_key, w_default):
-        w_result = self.getitem(w_dict, w_key)
-        if w_result is not None:
-            return w_result
-        self.setitem(w_dict, w_key, w_default)
-        return w_default
-
-    def delitem(self, w_dict, w_key):
-        space = self.space
-        w_key_type = space.type(w_key)
-        if space.is_w(w_key_type, space.w_unicode):
-            key = self.space.str_w(w_key)
-            if not self.unerase(w_dict.dstorage).deldictvalue(space, key):
-                raise KeyError
-        else:
-            raise KeyError
-
-    def length(self, w_dict):
-        return len(self.unerase(w_dict.dstorage).dict_w)
-
-    def w_keys(self, w_dict):
-        space = self.space
-        w_type = self.unerase(w_dict.dstorage)
-        return space.newlist([_wrapkey(space, key)
-                              for key in w_type.dict_w.iterkeys()])
-
-    def values(self, w_dict):
-        return [unwrap_cell(self.space, w_value) for w_value in self.unerase(w_dict.dstorage).dict_w.itervalues()]
-
-    def items(self, w_dict):
-        space = self.space
-        w_type = self.unerase(w_dict.dstorage)
-        return [space.newtuple([_wrapkey(space, key),
-                                unwrap_cell(space, w_value)])
-                for (key, w_value) in w_type.dict_w.iteritems()]
-
-    def clear(self, w_dict):
-        space = self.space
-        w_type = self.unerase(w_dict.dstorage)
-        if not w_type.is_heaptype():
-            raise oefmt(space.w_TypeError,
-                        "can't clear dictionary of type '%N'", w_type)
-        w_type.dict_w.clear()
-        w_type.mutated(None)
-
-    def getiterkeys(self, w_dict):
-        return self.unerase(w_dict.dstorage).dict_w.iterkeys()
-    def getitervalues(self, w_dict):
-        return self.unerase(w_dict.dstorage).dict_w.itervalues()
-    def getiteritems_with_hash(self, w_dict):
-        return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w)
-    def wrapkey(space, key):
-        return _wrapkey(space, key)
-    def wrapvalue(space, value):
-        return unwrap_cell(space, value)
-
-def _wrapkey(space, key):
-    # keys are utf-8 encoded identifiers from type's dict_w
-    return space.wrap(key.decode('utf-8'))
-
-create_iterator_classes(DictProxyStrategy)
-
-
-class MappingProxyStrategy(DictStrategy):
-    """Wraps an applevel mapping in a read-only dictionary."""
-    erase, unerase = rerased.new_erasing_pair("mappingproxy")
-    erase = staticmethod(erase)
-    unerase = staticmethod(unerase)
-
-    def getitem(self, w_dict, w_key):
-        try:
-            return self.space.getitem(self.unerase(w_dict.dstorage), w_key)
-        except OperationError as e:
-            if not e.match(self.space, self.space.w_KeyError):
-                raise
-            return None
-
-    def setitem(self, w_dict, w_key, w_value):
-        raise oefmt(self.space.w_TypeError,
-                    "'%T' object does not support item assignment", w_dict)
-
-    def delitem(self, w_dict, w_key):
-        raise oefmt(self.space.w_TypeError,
-                    "'%T' object does not support item deletion", w_dict)
-
-    def length(self, w_dict):
-        return self.space.len_w(self.unerase(w_dict.dstorage))
-
-    def getiterkeys(self, w_dict):
-        return self.space.iter(
-            self.space.call_method(self.unerase(w_dict.dstorage), "keys"))
-
-    def getitervalues(self, w_dict):
-        return self.space.iter(
-            self.space.call_method(self.unerase(w_dict.dstorage), "values"))
-
-    def getiteritems_with_hash(self, w_dict):
-        return self.space.iter(
-            self.space.call_method(self.unerase(w_dict.dstorage), "items"))
-
-    @staticmethod
-    def override_next_key(iterkeys):
-        w_keys = iterkeys.iterator
-        return iterkeys.space.next(w_keys)
-
-    @staticmethod
-    def override_next_value(itervalues):
-        w_values = itervalues.iterator
-        return itervalues.space.next(w_values)
-
-    @staticmethod
-    def override_next_item(iteritems):
-        w_items = iteritems.iterator
-        w_item = iteritems.space.next(w_items)
-        w_key, w_value = iteritems.space.unpackiterable(w_item, 2)
-        return w_key, w_value
-
-    def clear(self, w_dict):
-        raise oefmt(self.space.w_AttributeError, "clear")
-
-    def copy(self, w_dict):
-        return self.space.call_method(self.unerase(w_dict.dstorage), "copy")
-
-create_iterator_classes(
-    MappingProxyStrategy,
-    override_next_key=MappingProxyStrategy.override_next_key,
-    override_next_value=MappingProxyStrategy.override_next_value,
-    override_next_item=MappingProxyStrategy.override_next_item)
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -9,42 +9,20 @@
         assert 'a' in NotEmpty.__dict__
         assert 'a' in NotEmpty.__dict__.keys()
         assert 'b' not in NotEmpty.__dict__
-        NotEmpty.__dict__['b'] = 4
-        assert NotEmpty.b == 4
-        del NotEmpty.__dict__['b']
         assert NotEmpty.__dict__.get("b") is None
+        raises(TypeError, "NotEmpty.__dict__['b'] = 4")
         raises(TypeError, 'NotEmpty.__dict__[15] = "y"')
-        raises(KeyError, 'del NotEmpty.__dict__[15]')
+        raises(TypeError, 'del NotEmpty.__dict__[15]')
 
-        assert NotEmpty.__dict__.setdefault("string", 1) == 1
-        assert NotEmpty.__dict__.setdefault("string", 2) == 1
-        assert NotEmpty.string == 1
-        raises(TypeError, 'NotEmpty.__dict__.setdefault(15, 1)')
-
-    def test_dictproxy_popitem(self):
-        class A(object):
-            a = 42
-        seen = 0
-        try:
-            while True:
-                key, value = A.__dict__.popitem()
-                if key == 'a':
-                    assert value == 42
-                    seen += 1
-        except KeyError:
-            pass
-        assert seen == 1
+        raises(AttributeError, 'NotEmpty.__dict__.setdefault')
 
     def test_dictproxy_getitem(self):
         class NotEmpty(object):
             a = 1
         assert 'a' in NotEmpty.__dict__
-        class substr(str): pass
+        class substr(str):
+            pass
         assert substr('a') in NotEmpty.__dict__
-        # the following are only for py2
-        ## assert u'a' in NotEmpty.__dict__
-        ## assert NotEmpty.__dict__[u'a'] == 1
-        ## assert u'\xe9' not in NotEmpty.__dict__
 
     def test_dictproxyeq(self):
         class a(object):
@@ -63,9 +41,9 @@
         class a(object):
             pass
         s1 = repr(a.__dict__)
+        assert s1.startswith('mappingproxy({') and s1.endswith('})')
         s2 = str(a.__dict__)
-        assert s1 == s2
-        assert s1.startswith('mappingproxy({') and s1.endswith('})')
+        assert s1 == 'mappingproxy(%s)' % s2
 
     def test_immutable_dict_on_builtin_type(self):
         raises(TypeError, "int.__dict__['a'] = 1")
@@ -100,4 +78,3 @@
 
 class AppTestUserObjectMethodCache(AppTestUserObject):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
-
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -185,12 +185,12 @@
             skip("cannot run this test as apptest")
         for u in [u"", u"a", u"aa"]:
             assert id(self.unwrap_wrap_unicode(u)) == id(u)
-            s = str(u)
-            assert id(self.unwrap_wrap_str(s)) == id(s)
+            s = u.encode()
+            assert id(self.unwrap_wrap_bytes(s)) == id(s)
         #
-        assert id('') == (256 << 4) | 11     # always
+        assert id(b'') == (256 << 4) | 11     # always
         assert id(u'') == (257 << 4) | 11
-        assert id('a') == (ord('a') << 4) | 11
+        assert id(b'a') == (ord('a') << 4) | 11
         assert id(u'\u1234') == ((~0x1234) << 4) | 11
 
     def test_id_of_tuples(self):
@@ -243,13 +243,13 @@
         l = []
         def add(s, u):
             l.append(s)
-            l.append(self.unwrap_wrap_str(s))
+            l.append(self.unwrap_wrap_bytes(s))
             l.append(s[:1] + s[1:])
             l.append(u)
             l.append(self.unwrap_wrap_unicode(u))
             l.append(u[:1] + u[1:])
         for i in range(3, 18):
-            add(str(i), unicode(i))
+            add(str(i).encode(), str(i))
         add(b"s", u"s")
         add(b"", u"")
 
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -970,7 +970,6 @@
         raises(TypeError, setattr, list, 'foobar', 42)
         raises(TypeError, delattr, dict, 'keys')
         raises(TypeError, 'int.__dict__["a"] = 1')
-        raises(TypeError, 'int.__dict__.clear()')
 
     def test_nontype_in_mro(self):
         class OldStyle:
@@ -1028,10 +1027,9 @@
             pass
 
         a = A()
+        d = A.__dict__
         A.x = 1
-        assert A.__dict__["x"] == 1
-        A.__dict__['x'] = 5
-        assert A.x == 5
+        assert d["x"] == 1
 
     def test_we_already_got_one_1(self):
         # Issue #2079: highly obscure: CPython complains if we say
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -4,8 +4,8 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import (
     Function, StaticMethod, ClassMethod, FunctionWithFixedCode)
-from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\
-     descr_get_dict, dict_descr, Member, TypeDef
+from pypy.interpreter.typedef import (
+    weakref_descr, GetSetProperty, dict_descr, Member, TypeDef)
 from pypy.interpreter.astcompiler.misc import mangle
 from pypy.module.__builtin__ import abstractinst
 
@@ -344,7 +344,7 @@
     def deldictvalue(self, space, key):
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        if not self.is_heaptype():
+        if not (self.is_heaptype() or self.is_cpytype()):
             raise oefmt(space.w_TypeError,
                         "can't delete attributes on type object '%N'", self)
         try:
@@ -483,14 +483,14 @@
                 self.getdictvalue(self.space, attr)
             del self.lazyloaders
 
-    def getdict(self, space): # returning a dict-proxy!
-        from pypy.objspace.std.dictproxyobject import DictProxyStrategy
-        from pypy.objspace.std.dictproxyobject import W_DictProxyObject
+    def getdict(self, space):
+        from pypy.objspace.std.classdict import ClassDictStrategy
+        from pypy.objspace.std.dictmultiobject import W_DictObject
         if self.lazyloaders:
             self._cleanup_()    # force un-lazification
-        strategy = space.fromcache(DictProxyStrategy)
+        strategy = space.fromcache(ClassDictStrategy)
         storage = strategy.erase(self)
-        return W_DictProxyObject(space, strategy, storage)
+        return W_DictObject(space, strategy, storage)
 
     def is_heaptype(self):
         return self.flag_heaptype
@@ -929,6 +929,13 @@
     return space.newbool(
         abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj))
 
+def type_get_dict(space, w_cls):
+    from pypy.objspace.std.dictproxyobject import W_DictProxyObject
+    w_dict = w_cls.getdict(space)
+    if w_dict is None:
+        return space.w_None
+    return W_DictProxyObject(w_dict)
+
 W_TypeObject.typedef = TypeDef("type",
     __new__ = gateway.interp2app(descr__new__),
     __name__ = GetSetProperty(descr_get__name__, descr_set__name__),
@@ -936,7 +943,7 @@
     __bases__ = GetSetProperty(descr_get__bases__, descr_set__bases__),
     __base__ = GetSetProperty(descr__base),
     __mro__ = GetSetProperty(descr_get__mro__),
-    __dict__ = GetSetProperty(descr_get_dict),
+    __dict__=GetSetProperty(type_get_dict),
     __doc__ = GetSetProperty(descr__doc, descr_set__doc),
     __dir__ = gateway.interp2app(descr__dir),
     mro = gateway.interp2app(descr_mro),

From pypy.commits at gmail.com  Thu Aug 11 10:36:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 11 Aug 2016 07:36:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: remove always_inline from function,
 and change it to a class function
Message-ID: <57ac8d81.a111c20a.22d77.09e6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async
Changeset: r86142:511f8024c517
Date: 2016-08-11 16:34 +0200
http://bitbucket.org/pypy/pypy/changeset/511f8024c517/

Log:	remove always_inline from function, and change it to a class
	function

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -53,18 +53,6 @@
     else:
         return " object";
 
- at always_inline
-def list_unpack_helper(frame, itemcount):
-    space = frame.space
-    w_sum = space.newlist([], sizehint=itemcount)
-    for i in range(itemcount, 0, -1):
-        w_item = frame.peekvalue(i-1)
-        w_sum.extend(w_item)
-    while itemcount != 0:
-        frame.popvalue()
-        itemcount -= 1
-    return w_sum
-
 
 opcodedesc = bytecode_spec.opcodedesc
 HAVE_ARGUMENT = bytecode_spec.HAVE_ARGUMENT
@@ -1388,13 +1376,26 @@
             itemcount -= 1
         self.pushvalue(w_sum)
 
+    @jit.unroll_safe
+    def list_unpack_helper(frame, itemcount):
+        space = frame.space
+        w_sum = space.newlist([], sizehint=itemcount)
+        for i in range(itemcount, 0, -1):
+            w_item = frame.peekvalue(i-1)
+            w_sum.extend(w_item)
+        while itemcount != 0:
+            frame.popvalue()
+            itemcount -= 1
+        return w_sum
+
+
     def BUILD_TUPLE_UNPACK(self, itemcount, next_instr):
-        w_list = list_unpack_helper(self, itemcount)
+        w_list = self.list_unpack_helper(itemcount)
         items = [w_obj for w_obj in w_list.getitems_unroll()]
         self.pushvalue(self.space.newtuple(items))
 
     def BUILD_LIST_UNPACK(self, itemcount, next_instr):
-        w_sum = list_unpack_helper(self, itemcount)
+        w_sum = self.list_unpack_helper(itemcount)
         self.pushvalue(w_sum)
 
     def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr):

From pypy.commits at gmail.com  Thu Aug 11 11:00:38 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 11 Aug 2016 08:00:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: add unroll_safe to opcode dispatch
 function (loop inside)
Message-ID: <57ac9316.e2efc20a.985c1.0eeb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async
Changeset: r86143:1e95da517f26
Date: 2016-08-11 16:59 +0200
http://bitbucket.org/pypy/pypy/changeset/1e95da517f26/

Log:	add unroll_safe to opcode dispatch function (loop inside)

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1388,7 +1388,7 @@
             itemcount -= 1
         return w_sum
 
-
+    @jit.unroll_safe
     def BUILD_TUPLE_UNPACK(self, itemcount, next_instr):
         w_list = self.list_unpack_helper(itemcount)
         items = [w_obj for w_obj in w_list.getitems_unroll()]

From pypy.commits at gmail.com  Thu Aug 11 11:46:29 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 11 Aug 2016 08:46:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57ac9dd5.262ec20a.a0119.2967@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86144:b053ff5c2d6d
Date: 2016-08-11 16:19 +0100
http://bitbucket.org/pypy/pypy/changeset/b053ff5c2d6d/

Log:	hg merge default

diff too long, truncating to 2000 out of 4035 lines

diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -105,3 +105,26 @@
 .. branch: ep2016sprint
 
 Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -114,8 +114,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1307,3 +1307,22 @@
             counts = self.count_instructions(source)
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
+
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1801,6 +1801,40 @@
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
         """)
 
+    def resource_warning(self, w_msg, w_tb):
+        self.appexec([w_msg, w_tb],
+                     """(msg, tb):
+            import sys
+            print >> sys.stderr, msg
+            if tb:
+                print >> sys.stderr, "Created at (most recent call last):"
+                print >> sys.stderr, tb
+        """)
+
+    def format_traceback(self):
+        # we need to disable track_resources before calling the traceback
+        # module. Else, it tries to open more files to format the traceback,
+        # the file constructor will call space.format_traceback etc., in an
+        # inifite recursion
+        flag = self.sys.track_resources
+        self.sys.track_resources = False
+        try:
+            return self.appexec([],
+                         """():
+                import sys, traceback
+                # the "1" is because we don't want to show THIS code
+                # object in the traceback
+                try:
+                    f = sys._getframe(1)
+                except ValueError:
+                    # this happens if you call format_traceback at the very beginning
+                    # of startup, when there is no bottom code object
+                    return '<no stacktrace available>'
+                return "".join(traceback.format_stack(f))
+            """)
+        finally:
+            self.sys.track_resources = flag
+
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -209,6 +209,13 @@
         self.check(['-c', 'pass'], {'PYTHONNOUSERSITE': '1'}, sys_argv=['-c'],
                    run_command='pass', **expected)
 
+    def test_track_resources(self, monkeypatch):
+        myflag = [False]
+        def pypy_set_track_resources(flag):
+            myflag[0] = flag
+        monkeypatch.setattr(sys, 'pypy_set_track_resources', pypy_set_track_resources, raising=False)
+        self.check(['-X', 'track-resources'], {}, sys_argv=[''], run_stdin=True)
+        assert myflag[0] == True
 
 class TestInteraction:
     """
@@ -1152,4 +1159,3 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -458,3 +458,28 @@
         space.finish()
         # assert that we reach this point without getting interrupted
         # by the OperationError(NameError)
+
+    def test_format_traceback(self):
+        from pypy.tool.pytest.objspace import maketestobjspace
+        from pypy.interpreter.gateway import interp2app
+        #
+        def format_traceback(space):
+            return space.format_traceback()
+        #
+        space = maketestobjspace()
+        w_format_traceback = space.wrap(interp2app(format_traceback))
+        w_tb = space.appexec([w_format_traceback], """(format_traceback):
+            def foo():
+                return bar()
+            def bar():
+                return format_traceback()
+            return foo()
+        """)
+        tb = space.str_w(w_tb)
+        expected = '\n'.join([
+            '  File "?", line 6, in anonymous',  # this is the appexec code object
+            '  File "?", line 3, in foo',
+            '  File "?", line 5, in bar',
+            ''
+        ])
+        assert tb == expected
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.7.0"
+VERSION = "1.8.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -157,11 +157,13 @@
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
         try:
+            keepalives = [None] * len(args_w)    # None or strings
             for i in range(len(args_w)):
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                 w_obj = args_w[i]
                 argtype = self.fargs[i]
-                if argtype.convert_argument_from_object(data, w_obj):
+                if argtype.convert_argument_from_object(data, w_obj,
+                                                        keepalives, i):
                     # argtype is a pointer type, and w_obj a list/tuple/str
                     mustfree_max_plus_1 = i + 1
 
@@ -177,9 +179,13 @@
                 if isinstance(argtype, W_CTypePointer):
                     data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                     flag = get_mustfree_flag(data)
+                    raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                     if flag == 1:
-                        raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
+                    elif flag >= 4:
+                        value = keepalives[i]
+                        assert value is not None
+                        rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
             lltype.free(buffer, flavor='raw')
             keepalive_until_here(args_w)
         return w_res
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -83,7 +83,7 @@
         raise oefmt(space.w_TypeError, "cannot initialize cdata '%s'",
                     self.name)
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         self.convert_from_object(cdata, w_ob)
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -16,8 +16,8 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
+    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
     length = -1
 
     def __init__(self, space, size, extra, extra_position, ctitem,
@@ -30,6 +30,9 @@
         #  - for functions, it is the return type
         self.ctitem = ctitem
         self.can_cast_anything = could_cast_anything and ctitem.cast_anything
+        self.accept_str = (self.can_cast_anything or
+                            (ctitem.is_primitive_integer and
+                             ctitem.size == rffi.sizeof(lltype.Char)))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -72,9 +75,7 @@
                 pass
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
-        elif (self.can_cast_anything or
-              (self.ctitem.is_primitive_integer and
-               self.ctitem.size == rffi.sizeof(lltype.Char))):
+        elif self.accept_str:
             if not space.isinstance_w(w_ob, space.w_str):
                 raise self._convert_error("bytes or list or tuple", w_ob)
             s = space.str_w(w_ob)
@@ -262,8 +263,16 @@
         else:
             return lltype.nullptr(rffi.CCHARP.TO)
 
-    def _prepare_pointer_call_argument(self, w_init, cdata):
+    def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
         space = self.space
+        if self.accept_str and space.isinstance_w(w_init, space.w_str):
+            # special case to optimize strings passed to a "char *" argument
+            value = w_init.str_w(space)
+            keepalives[i] = value
+            buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
+            rffi.cast(rffi.CCHARPP, cdata)[0] = buf
+            return ord(buf_flag)    # 4, 5 or 6
+        #
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
@@ -300,10 +309,11 @@
         rffi.cast(rffi.CCHARPP, cdata)[0] = result
         return 1
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
         result = (not isinstance(w_ob, cdataobj.W_CData) and
-                  self._prepare_pointer_call_argument(w_ob, cdata))
+                  self._prepare_pointer_call_argument(w_ob, cdata,
+                                                      keepalives, i))
         if result == 0:
             self.convert_from_object(cdata, w_ob)
         set_mustfree_flag(cdata, result)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -353,7 +353,7 @@
 'array.array' or numpy arrays."""
         #
         w_ctchara = newtype._new_chara_type(self.space)
-        return func.from_buffer(self.space, w_ctchara, w_python_buffer)
+        return func._from_buffer(self.space, w_ctchara, w_python_buffer)
 
 
     @unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated
+from rpython.rlib import jit
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -132,17 +133,66 @@
         raise oefmt(space.w_TypeError,
                     "needs 'char[]', got '%s'", w_ctype.name)
     #
+    return _from_buffer(space, w_ctype, w_x)
+
+def _from_buffer(space, w_ctype, w_x):
     buf = _fetch_as_read_buffer(space, w_x)
-    try:
-        _cdata = buf.get_raw_address()
-    except ValueError:
-        raise oefmt(space.w_TypeError,
-                    "from_buffer() got a '%T' object, which supports the "
-                    "buffer interface but cannot be rendered as a plain "
-                    "raw address on PyPy", w_x)
+    if space.isinstance_w(w_x, space.w_str):
+        _cdata = get_raw_address_of_string(space, w_x)
+    else:
+        try:
+            _cdata = buf.get_raw_address()
+        except ValueError:
+            raise oefmt(space.w_TypeError,
+                        "from_buffer() got a '%T' object, which supports the "
+                        "buffer interface but cannot be rendered as a plain "
+                        "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
 
+# ____________________________________________________________
+
+class RawBytes(object):
+    def __init__(self, string):
+        self.ptr = rffi.str2charp(string, track_allocation=False)
+    def __del__(self):
+        rffi.free_charp(self.ptr, track_allocation=False)
+
+class RawBytesCache(object):
+    def __init__(self, space):
+        from pypy.interpreter.baseobjspace import W_Root
+        from rpython.rlib import rweakref
+        self.wdict = rweakref.RWeakKeyDictionary(W_Root, RawBytes)
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(space, w_x):
+    """Special case for ffi.from_buffer(string).  Returns a 'char *' that
+    is valid as long as the string object is alive.  Two calls to
+    ffi.from_buffer(same_string) are guaranteed to return the same pointer.
+    """
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rtyper.lltypesystem.rstr import STR
+    from rpython.rtyper.lltypesystem import llmemory
+    from rpython.rlib import rgc
+
+    cache = space.fromcache(RawBytesCache)
+    rawbytes = cache.wdict.get(w_x)
+    if rawbytes is None:
+        data = space.str_w(w_x)
+        if we_are_translated() and not rgc.can_move(data):
+            lldata = llstr(data)
+            data_start = (llmemory.cast_ptr_to_adr(lldata) +
+                          rffi.offsetof(STR, 'chars') +
+                          llmemory.itemoffsetof(STR.chars, 0))
+            data_start = rffi.cast(rffi.CCHARP, data_start)
+            data_start[len(data)] = '\x00'   # write the final extra null
+            return data_start
+        rawbytes = RawBytes(data)
+        cache.wdict.set(w_x, rawbytes)
+    return rawbytes.ptr
+
+# ____________________________________________________________
+
 
 def unsafe_escaping_ptr_for_ptr_or_array(w_cdata):
     if not w_cdata.ctype.is_nonfunc_pointer_or_array:
diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py
--- a/pypy/module/_cffi_backend/parse_c_type.py
+++ b/pypy/module/_cffi_backend/parse_c_type.py
@@ -97,11 +97,8 @@
                                   [rffi.INT], rffi.CCHARP)
 
 def parse_c_type(info, input):
-    p_input = rffi.str2charp(input)
-    try:
+    with rffi.scoped_view_charp(input) as p_input:
         res = ll_parse_c_type(info, p_input)
-    finally:
-        rffi.free_charp(p_input)
     return rffi.cast(lltype.Signed, res)
 
 NULL_CTX = lltype.nullptr(PCTX.TO)
@@ -130,15 +127,13 @@
     return rffi.getintfield(src_ctx, 'c_num_types')
 
 def search_in_globals(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_globals(ctx, c_name,
-                                  rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_globals(ctx, c_name,
+                                      rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
 
 def search_in_struct_unions(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_struct_unions(ctx, c_name,
-                                        rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_struct_unions(ctx, c_name,
+                                            rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3330,13 +3330,18 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharA = new_array_type(BCharP, None)
-    py.test.raises(TypeError, from_buffer, BCharA, b"foo")
+    p1 = from_buffer(BCharA, b"foo")
+    assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # from_buffer(buffer(b"foo")) does not work, because it's not
+        # implemented on pypy; only from_buffer(b"foo") works.
         py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
         py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
     try:
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -402,21 +402,20 @@
             _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
         from rpython.rlib import rwin32
 
-        charp = rffi.str2charp(buf)
-        written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                    flavor='raw')
-        try:
-            result = _WriteFile(
-                self.handle, rffi.ptradd(charp, offset),
-                size, written_ptr, rffi.NULL)
+        with rffi.scoped_view_charp(buf) as charp:
+            written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
+                                        flavor='raw')
+            try:
+                result = _WriteFile(
+                    self.handle, rffi.ptradd(charp, offset),
+                    size, written_ptr, rffi.NULL)
 
-            if (result == 0 and
-                rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
-                raise oefmt(space.w_ValueError,
-                            "Cannot send %d bytes over connection", size)
-        finally:
-            rffi.free_charp(charp)
-            lltype.free(written_ptr, flavor='raw')
+                if (result == 0 and
+                    rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
+                    raise oefmt(space.w_ValueError,
+                                "Cannot send %d bytes over connection", size)
+            finally:
+                lltype.free(written_ptr, flavor='raw')
 
     def do_recv_string(self, space, buflength, maxlength):
         from pypy.module._multiprocessing.interp_win32 import (
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,6 +1,7 @@
 import sys, os
-import py
+import pytest
 from pypy.tool.pytest.objspace import gettestobjspace
+from pypy.interpreter.gateway import interp2app
 from rpython.tool.udir import udir
 from rpython.rlib import rsocket
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -13,8 +14,6 @@
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
     mod.path = udir.join('fd')
     mod.path.write('fo')
-    mod.raises = py.test.raises # make raises available from app-level tests
-    mod.skip = py.test.skip
 
 def test_gethostname():
     host = space.appexec([w_socket], "(_socket): return _socket.gethostname()")
@@ -42,7 +41,7 @@
     for host in ["localhost", "127.0.0.1", "::1"]:
         if host == "::1" and not ipv6:
             from pypy.interpreter.error import OperationError
-            with py.test.raises(OperationError):
+            with pytest.raises(OperationError):
                 space.appexec([w_socket, space.wrap(host)],
                               "(_socket, host): return _socket.gethostbyaddr(host)")
             continue
@@ -58,14 +57,14 @@
     assert space.unwrap(port) == 25
     # 1 arg version
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyname second argument is not optional before python 2.4")
+        pytest.skip("getservbyname second argument is not optional before python 2.4")
     port = space.appexec([w_socket, space.wrap(name)],
                         "(_socket, name): return _socket.getservbyname(name)")
     assert space.unwrap(port) == 25
 
 def test_getservbyport():
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyport does not exist before python 2.4")
+        pytest.skip("getservbyport does not exist before python 2.4")
     port = 25
     # 2 args version
     name = space.appexec([w_socket, space.wrap(port)],
@@ -139,7 +138,7 @@
 
 def test_pton_ntop_ipv4():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     tests = [
         ("123.45.67.89", "\x7b\x2d\x43\x59"),
         ("0.0.0.0", "\x00" * 4),
@@ -155,9 +154,9 @@
 
 def test_ntop_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -176,9 +175,9 @@
 
 def test_pton_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -197,7 +196,7 @@
         assert space.unwrap(w_packed) == packed
 
 def test_has_ipv6():
-    py.test.skip("has_ipv6 is always True on PyPy for now")
+    pytest.skip("has_ipv6 is always True on PyPy for now")
     res = space.appexec([w_socket], "(_socket): return _socket.has_ipv6")
     assert space.unwrap(res) == socket.has_ipv6
 
@@ -231,7 +230,7 @@
 def test_addr_raw_packet():
     from pypy.module._socket.interp_socket import addr_as_object
     if not hasattr(rsocket._c, 'sockaddr_ll'):
-        py.test.skip("posix specific test")
+        pytest.skip("posix specific test")
     # HACK: To get the correct interface number of lo, which in most cases is 1,
     # but can be anything (i.e. 39), we need to call the libc function
     # if_nametoindex to get the correct index
@@ -653,11 +652,11 @@
 class AppTestNetlink:
     def setup_class(cls):
         if not hasattr(os, 'getpid'):
-            py.test.skip("AF_NETLINK needs os.getpid()")
+            pytest.skip("AF_NETLINK needs os.getpid()")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_NETLINK')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_NETLINK on this platform")
+            pytest.skip("no AF_NETLINK on this platform")
         cls.space = space
 
     def test_connect_to_kernel_netlink_routing_socket(self):
@@ -673,11 +672,11 @@
 class AppTestPacket:
     def setup_class(cls):
         if not hasattr(os, 'getuid') or os.getuid() != 0:
-            py.test.skip("AF_PACKET needs to be root for testing")
+            pytest.skip("AF_PACKET needs to be root for testing")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_PACKET')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_PACKET on this platform")
+            pytest.skip("no AF_PACKET on this platform")
         cls.space = space
 
     def test_convert_between_tuple_and_sockaddr_ll(self):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -146,7 +146,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
@@ -158,7 +158,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
@@ -192,7 +192,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
@@ -204,7 +204,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
@@ -239,7 +239,7 @@
 
         Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
         bound on the entropy contained in string."""
-        with rffi.scoped_str2charp(string) as buf:
+        with rffi.scoped_nonmovingbuffer(string) as buf:
             libssl_RAND_add(buf, len(string), entropy)
 
     def _RAND_bytes(space, n, pseudo):
diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -537,9 +537,8 @@
     releasegil=ts_helper,
     compilation_info=backend.eci)
 def c_charp2stdstring(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2stdstring(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2stdstring(charp)
     return result
 _c_stdstring2stdstring = rffi.llexternal(
     "cppyy_stdstring2stdstring",
diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -82,9 +82,8 @@
     releasegil=ts_helper,
     compilation_info=eci)
 def c_charp2TString(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2TString(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2TString(charp)
     return result
 _c_TString2TString = rffi.llexternal(
     "cppyy_TString2TString",
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -65,6 +65,7 @@
                 else:    # only other use is sring
                     n = len(obj._string)
                     assert raw_string == rffi.cast(rffi.CCHARP, 0)
+                    # XXX could use rffi.get_nonmovingbuffer_final_null()
                     raw_string = rffi.str2charp(obj._string)
                     data = rffi.cast(rffi.CCHARPP, data)
                     data[0] = raw_string
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,6 +11,9 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+        c_fdopen, c_fileno,
+        c_fopen)# for tests
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import NameManager
@@ -84,44 +87,32 @@
 assert CONST_WSTRING == rffi.CWCHARP
 
 # FILE* interface
-FILEP = rffi.COpaquePtr('FILE')
 
 if sys.platform == 'win32':
     dash = '_'
 else:
     dash = ''
-fileno = rffi.llexternal(dash + 'fileno', [FILEP], rffi.INT)
-fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
-fdopen = rffi.llexternal(dash + 'fdopen', [rffi.INT, CONST_STRING],
-                  FILEP, save_err=rffi.RFFI_SAVE_ERRNO)
 
-_fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
-    if not is_valid_fd(fileno(fp)):
+    if not is_valid_fd(c_fileno(fp)):
         return -1
-    return _fclose(fp)
+    return c_fclose(fp)
 
-_fwrite = rffi.llexternal('fwrite',
-                         [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                         rffi.SIZE_T)
 def fwrite(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fwrite(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fwrite(buf, sz, n, fp)
 
-_fread = rffi.llexternal('fread',
-                        [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                        rffi.SIZE_T)
 def fread(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fread(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fread(buf, sz, n, fp)
 
 _feof = rffi.llexternal('feof', [FILEP], rffi.INT)
 def feof(fp):
-    validate_fd(fileno(fp))
+    validate_fd(c_fileno(fp))
     return _feof(fp)
 
 def is_valid_fp(fp):
-    return is_valid_fd(fileno(fp))
+    return is_valid_fd(c_fileno(fp))
 
 pypy_decl = 'pypy_decl.h'
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -96,7 +96,8 @@
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
             py_str.c_ob_size, len(s))
-    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    with rffi.scoped_nonmovingbuffer(s) as s_ptr:
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
     py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
diff --git a/pypy/module/cpyext/c-api.txt b/pypy/module/cpyext/c-api.txt
deleted file mode 100644
--- a/pypy/module/cpyext/c-api.txt
+++ /dev/null
@@ -1,71 +0,0 @@
-Reference Count
-===============
-
-XXX
-
-Borrowed References
-===================
-
-XXX
-
-PyStringObject support
-======================
-
-The problem
------------
-
-PyString_AsString() returns a (non-movable) pointer to the underlying
-buffer, whereas pypy strings are movable.  C code may temporarily
-store this address and use it, as long as it owns a reference to the
-PyObject.  There is no "release" function to specify that the pointer
-is not needed any more.
-
-Note that the pointer may be used to fill the initial value of
-string. This is valid only when the string was just allocated, and is
-not used elsewhere.
-
-Proposed solution
------------------
-
-Our emulation of the PyStringObject contains an additional member: a
-pointer to a char buffer; it may be NULL.
-
-- A string allocated by pypy will be converted into a PyStringObject
-  with a NULL buffer.  When PyString_AsString() is called, memory is
-  allocated (with flavor='raw') and content is copied.
-
-- A string allocated with PyString_FromStringAndSize(NULL, size) will
-  allocate a buffer with the specified size, but the reference won't
-  be stored in the global map py_objects_r2w; there won't be a
-  corresponding object in pypy.  When from_ref() or Py_INCREF() is
-  called, the pypy string is created, and added in py_objects_r2w.
-  The buffer is then supposed to be immutable.
-
-- _PyString_Resize works only on not-yet-pypy'd strings, and returns a
-  similar object.
-
-- PyString_Size don't need to force the object. (in this case, another
-  "size" member is needed)
-
-- There could be an (expensive!) check in from_ref() that the buffer
-  still corresponds to the pypy gc-managed string.
-
-PySequence_Fast support
-======================
-There are five functions for fast sequence access offered by the CPython API:
-
-PyObject* PySequence_Fast(PyObject *o, const char *m)
-
-PyObject* PySequence_Fast_GET_ITEM(	PyObject *o, int i)
-
-PyObject** PySequence_Fast_ITEMS(	PyObject *o)
-
-PyObject* PySequence_ITEM(	PyObject *o, int i)
-
-int PySequence_Fast_GET_SIZE(	PyObject *o)
-
-PyPy supports four of these, but does not support PySequence_Fast_ITEMS.
-(Various ways to support PySequence_Fast_ITEMS were considered. They all had
-two things in common: they would have taken a lot of work, and they would have
-resulted in incomplete semantics or in poor performance. We decided that a slow
-implementation of PySequence_Fast_ITEMS was not very useful.)
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -25,6 +25,8 @@
                          flavor='raw',
                          add_memory_pressure=True)
 
+realloc = rffi.llexternal('realloc', [rffi.VOIDP, rffi.SIZE_T], rffi.VOIDP)
+
 @cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
@@ -32,7 +34,7 @@
                          flavor='raw',
                          add_memory_pressure=True)
     # XXX FIXME
-    return lltype.nullptr(rffi.VOIDP.TO)
+    return realloc(ptr, size)
 
 @cpython_api([rffi.VOIDP], lltype.Void)
 def PyObject_Free(space, ptr):
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -10,7 +10,7 @@
 from pypy.objspace.std import tupleobject
 
 from pypy.module.cpyext.tupleobject import PyTuple_Check, PyTuple_SetItem
-from pypy.module.cpyext.object import Py_IncRef, Py_DecRef
+from pypy.module.cpyext.pyobject import decref
 
 from pypy.module.cpyext.dictobject import PyDict_Check
 
@@ -252,7 +252,7 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        Py_DecRef(w_list.space, storage._elems[index])
+        decref(w_list.space, storage._elems[index])
         storage._elems[index] = make_ref(w_list.space, w_obj)
 
     def length(self, w_list):
@@ -264,9 +264,8 @@
         return storage._elems
 
     def getslice(self, w_list, start, stop, step, length):
-        #storage = self.unerase(w_list.lstorage)
-        raise oefmt(w_list.space.w_NotImplementedError,
-                    "settting a slice of a PySequence_Fast is not supported")
+        w_list.switch_to_object_strategy()
+        return w_list.strategy.getslice(w_list, start, stop, step, length)
 
     def getitems(self, w_list):
         # called when switching list strategy, so convert storage
@@ -389,5 +388,5 @@
 
     def __del__(self):
         for i in range(self._length):
-            Py_DecRef(self.space, self._elems[i])
+            decref(self.space, self._elems[i])
         lltype.free(self._elems, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.eval import (
     Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP
+from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
@@ -130,19 +130,19 @@
     def test_run_file(self, space, api):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
-        fp = fopen(str(filepath), "rb")
+        fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_ZeroDivisionError
         api.PyErr_Clear()
 
         # try again, but with a closed file
-        fp = fopen(str(filepath), "rb")
-        os.close(fileno(fp))
+        fp = c_fopen(str(filepath), "rb")
+        os.close(c_fileno(fp))
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -212,8 +212,9 @@
         assert type(x) is float
         assert x == -12.34
 
-    @pytest.mark.skipif(True, reason='realloc not fully implemented')
     def test_object_realloc(self):
+        if not self.runappdirect:
+            skip('no untranslated support for realloc')
         module = self.import_extension('foo', [
             ("realloctest", "METH_NOARGS",
              """
@@ -221,12 +222,11 @@
                  char *copy, *orig = PyObject_MALLOC(12);
                  memcpy(orig, "hello world", 12);
                  copy = PyObject_REALLOC(orig, 15);
+                 /* realloc() takes care of freeing orig, if changed */
                  if (copy == NULL)
                      Py_RETURN_NONE;
                  ret = PyBytes_FromStringAndSize(copy, 12);
-                 if (copy != orig)
-                     PyObject_Free(copy);
-                 PyObject_Free(orig);
+                 PyObject_Free(copy);
                  return ret;
              """)])
         x = module.realloctest()
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -78,6 +78,17 @@
         assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
         assert space.eq_w(w_t, space.wrap([1, 3, 5]))
 
+    def test_get_slice_fast(self, space, api):
+        w_t = space.wrap([1, 2, 3, 4, 5])
+        api.PySequence_Fast(w_t, "foo") # converts
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4]
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4]
+
+        assert api.PySequence_DelSlice(w_t, 1, 4) == 0
+        assert space.eq_w(w_t, space.wrap([1, 5]))
+        assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
+        assert space.eq_w(w_t, space.wrap([1, 3, 5]))
+
     def test_iter(self, space, api):
         w_t = space.wrap((1, 2))
         w_iter = api.PySeqIter_New(w_t)
@@ -226,18 +237,33 @@
         assert space.int_w(space.len(w_l)) == 10
 
 
-class XAppTestSequenceObject(AppTestCpythonExtensionBase):
-    def test_sequenceobject(self):
+class AppTestSequenceObject(AppTestCpythonExtensionBase):
+    def test_fast(self):
         module = self.import_extension('foo', [
             ("test_fast_sequence", "METH_VARARGS",
              """
-                PyObject * o = PyTuple_GetItem(args, 0);
+                int size, i;
+                PyTypeObject * common_type;
+                PyObject *foo, **objects;
+                PyObject * seq = PyTuple_GetItem(args, 0);
                 /* XXX assert it is a tuple */
-                PyObject *foo = PySequence_Fast(o, "some string");
-                PyObject ** res = PySequence_Fast_ITEMS(foo);
-                /* XXX do some kind of test on res */
-                /* XXX now what? who manages res's refcount? */
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "some string");
+                objects = PySequence_Fast_ITEMS(foo);
+                size = PySequence_Fast_GET_SIZE(seq);
+                common_type = size > 0 ? Py_TYPE(objects[0]) : NULL;
+                for (i = 1; i < size; ++i) {
+                    if (Py_TYPE(objects[i]) != common_type) {
+                        common_type = NULL;
+                        break;
+                    }
+                }
+                Py_DECREF(foo);
+                Py_DECREF(common_type);
                 return PyBool_FromLong(1);
              """)])
-        assert module.test_fast_sequence([1, 2, 3, 4])
+        s = [1, 2, 3, 4]
+        assert module.test_fast_sequence(s[0:-1])
+        assert module.test_fast_sequence(s[::-1])
 
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -47,6 +47,7 @@
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
+    from rpython.tool.error import offset2lineno
     bcindex = ord(bytecode.co_code[next_instr])
     opname = ""
     if 0 <= bcindex < len(opcode_method_names):
@@ -54,7 +55,8 @@
     name = bytecode.co_name
     if not name:
         name = ""
-    return (bytecode.co_filename, bytecode.co_firstlineno,
+    line = offset2lineno(bytecode, intmask(next_instr))
+    return (bytecode.co_filename, line,
             name, intmask(next_instr), opname)
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -19,6 +19,7 @@
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
         self.debug = True
+        self.track_resources = False
         self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
@@ -48,6 +49,8 @@
         '_current_frames'       : 'currentframes._current_frames',
         'setrecursionlimit'     : 'vm.setrecursionlimit',
         'getrecursionlimit'     : 'vm.getrecursionlimit',
+        'pypy_set_track_resources' : 'vm.set_track_resources',
+        'pypy_get_track_resources' : 'vm.get_track_resources',
         'setcheckinterval'      : 'vm.setcheckinterval',
         'getcheckinterval'      : 'vm.getcheckinterval',
         'exc_info'              : 'vm.exc_info',
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -61,6 +61,13 @@
     """
     return space.wrap(space.sys.recursionlimit)
 
+ at unwrap_spec(flag=bool)
+def set_track_resources(space, flag):
+    space.sys.track_resources = flag
+
+def get_track_resources(space):
+    return space.wrap(space.sys.track_resources)
+
 @unwrap_spec(interval=int)
 def setcheckinterval(space, interval):
     """Tell the Python interpreter to check for asynchronous events every
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -130,7 +130,7 @@
                 cls.module = str(udir.join('testownlib.dll'))
         else:
             subprocess.check_call(
-                'gcc testownlib.c -shared -fPIC -o testownlib.so',
+                'cc testownlib.c -shared -fPIC -o testownlib.so',
                 cwd=str(udir), shell=True)
             cls.module = str(udir.join('testownlib.so'))
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -852,9 +852,12 @@
     assert str(e2.value) == "foo0() takes no arguments (2 given)"
     assert str(e3.value) == "foo1() takes exactly one argument (0 given)"
     assert str(e4.value) == "foo1() takes exactly one argument (2 given)"
-    assert str(e5.value) == "foo2() takes exactly 2 arguments (0 given)"
-    assert str(e6.value) == "foo2() takes exactly 2 arguments (1 given)"
-    assert str(e7.value) == "foo2() takes exactly 2 arguments (3 given)"
+    assert str(e5.value) in ["foo2 expected 2 arguments, got 0",
+                             "foo2() takes exactly 2 arguments (0 given)"]
+    assert str(e6.value) in ["foo2 expected 2 arguments, got 1",
+                             "foo2() takes exactly 2 arguments (1 given)"]
+    assert str(e7.value) in ["foo2 expected 2 arguments, got 3",
+                             "foo2() takes exactly 2 arguments (3 given)"]
 
 def test_address_of_function():
     ffi = FFI()
@@ -1916,3 +1919,47 @@
     ffi.cdef("bool f(void);")
     lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
     assert lib.f() == 1
+
+def test_bool_in_cpp_2():
+    ffi = FFI()
+    ffi.cdef('int add(int a, int b);')
+    lib = verify(ffi, "test_bool_bug_cpp", '''
+        typedef bool _Bool;  /* there is a Windows header with this line */
+        int add(int a, int b)
+        {
+            return a + b;
+        }''', source_extension='.cpp')
+    c = lib.add(2, 3)
+    assert c == 5
+
+def test_struct_field_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[2]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+
+def test_function_arg_opaque():
+    py.test.skip("can currently declare a function with an opaque struct "
+                 "as argument, but AFAICT it's impossible to call it later")
+
+def test_function_returns_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a foo(int);")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_function_returns_opaque", "?")
+    assert str(e.value) == ("function foo: 'struct a' is used as result type,"
+                            " but is opaque")
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -133,6 +133,12 @@
         # You cannot assing character format codes as restype any longer
         raises(TypeError, setattr, f, "restype", "i")
 
+    def test_unicode_function_name(self):
+        f = dll[u'_testfunc_i_bhilfd']
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_int
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        assert result == 21
 
     def test_truncate_python_longs(self):
         f = dll._testfunc_i_bhilfd
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -434,4 +434,5 @@
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
 FakeObjSpace.sys.dlopenflags = 123
+FakeObjSpace.sys.track_resources = False
 FakeObjSpace.builtin = FakeModule()
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -930,6 +930,7 @@
         abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj))
 
 def type_get_dict(space, w_cls):
+    w_cls = _check(space, w_cls)
     from pypy.objspace.std.dictproxyobject import W_DictProxyObject
     w_dict = w_cls.getdict(space)
     if w_dict is None:
@@ -1287,8 +1288,8 @@
     cycle.append(candidate)
     cycle.reverse()
     names = [cls.getname(space) for cls in cycle]
-    raise OperationError(space.w_TypeError, space.wrap(
-        u"cycle among base classes: " + u' < '.join(names)))
+    raise oefmt(space.w_TypeError,
+                "cycle among base classes: %s", ' < '.join(names))
 
 
 class TypeCache(SpaceCache):
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -143,3 +143,5 @@
     def is_w(self, obj1, obj2):
         return obj1 is obj2
 
+    def setitem(self, obj, key, value):
+        obj[key] = value
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-# hypothesis is used for test generation on untranslated jit tests
+# hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -401,6 +401,9 @@
 class __extend__(pairtype(SomeString, SomeTuple),
                  pairtype(SomeUnicodeString, SomeTuple)):
     def mod((s_string, s_tuple)):
+        if not s_string.is_constant():
+            raise AnnotatorError("string formatting requires a constant "
+                                 "string/unicode on the left of '%'")
         is_string = isinstance(s_string, SomeString)
         is_unicode = isinstance(s_string, SomeUnicodeString)
         assert is_string or is_unicode
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4623,6 +4623,14 @@
         a = self.RPythonAnnotator()
         a.build_types(main, [int])
 
+    def test_string_mod_nonconstant(self):
+        def f(x):
+            return x % 5
+        a = self.RPythonAnnotator()
+        e = py.test.raises(AnnotatorError, a.build_types, f, [str])
+        assert ('string formatting requires a constant string/unicode'
+                in str(e.value))
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -883,6 +883,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                               self.cpu.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -280,7 +280,7 @@
     concrete_type = '\x00'
 
     def __init__(self, basesize, itemsize, lendescr, flag, is_pure=False, concrete_type='\x00'):
-        self.basesize = basesize
+        self.basesize = basesize    # this includes +1 for STR
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
@@ -676,7 +676,7 @@
 
 def unpack_arraydescr(arraydescr):
     assert isinstance(arraydescr, ArrayDescr)
-    ofs = arraydescr.basesize
+    ofs = arraydescr.basesize    # this includes +1 for STR
     size = arraydescr.itemsize
     sign = arraydescr.is_item_signed()
     return size, ofs, sign
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -293,6 +293,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_load_or_indexed(op, op.getarg(0), op.getarg(1),
                                          itemsize, itemsize, basesize, NOT_SIGNED)
         elif opnum == rop.UNICODEGETITEM:
@@ -304,6 +305,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_store_or_indexed(op, op.getarg(0), op.getarg(1), op.getarg(2),
                                          itemsize, itemsize, basesize)
         elif opnum == rop.UNICODESETITEM:
diff --git a/rpython/jit/backend/llsupport/symbolic.py b/rpython/jit/backend/llsupport/symbolic.py
--- a/rpython/jit/backend/llsupport/symbolic.py
+++ b/rpython/jit/backend/llsupport/symbolic.py
@@ -29,7 +29,7 @@
 def get_array_token(T, translate_support_code):
     # T can be an array or a var-sized structure
     if translate_support_code:
-        basesize = llmemory.sizeof(T, 0)
+        basesize = llmemory.sizeof(T, 0)     # this includes +1 for STR
         if isinstance(T, lltype.Struct):
             SUBARRAY = getattr(T, T._arrayfld)
             itemsize = llmemory.sizeof(SUBARRAY.OF)
@@ -57,6 +57,7 @@
             assert carray.length.size == WORD
             ofs_length = before_array_part + carray.length.offset
         basesize = before_array_part + carray.items.offset
+        basesize += T._hints.get('extra_item_after_alloc', 0)  # +1 for STR
         carrayitem = ll2ctypes.get_ctypes_type(T.OF)
         itemsize = ctypes.sizeof(carrayitem)
     return basesize, itemsize, ofs_length
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -435,8 +435,10 @@
 def test_bytearray_descr():
     c0 = GcCache(False)
     descr = get_array_descr(c0, rstr.STR)   # for bytearray
+    # note that we get a basesize that has 1 extra byte for the final null char
+    # (only for STR)
     assert descr.flag == FLAG_UNSIGNED
-    assert descr.basesize == struct.calcsize("PP")         # hash, length
+    assert descr.basesize == struct.calcsize("PP") + 1     # hash, length, extra
     assert descr.lendescr.offset == struct.calcsize("P")   # hash
     assert not descr.is_array_of_pointers()
 
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -647,6 +647,9 @@
         """)
 
     def test_rewrite_assembler_newstr_newunicode(self):
+        # note: strdescr.basesize already contains the extra final character,
+        # so that's why newstr(14) is rounded up to 'basesize+15' and not
+        # 'basesize+16'.
         self.check_rewrite("""
             [i2]
             p0 = newstr(14)
@@ -657,12 +660,12 @@
         """, """
             [i2]
             p0 = call_malloc_nursery(                                \
-                      %(strdescr.basesize + 16 * strdescr.itemsize + \
+                      %(strdescr.basesize + 15 * strdescr.itemsize + \
                         unicodedescr.basesize + 10 * unicodedescr.itemsize)d)
             gc_store(p0, 0,  %(strdescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p0, %(strlendescr.offset)s, 14, %(strlendescr.field_size)s)
             gc_store(p0, 0,  0, %(strhashdescr.field_size)s)
-            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
+            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 15 * strdescr.itemsize)d)
             gc_store(p1, 0,  %(unicodedescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p1, %(unicodelendescr.offset)s, 10, %(unicodelendescr.field_size)s)
             gc_store(p1, 0,  0, %(unicodehashdescr.field_size)s)
@@ -1240,14 +1243,14 @@
         #               'i3 = gc_load_i(p0,i5,%(unicodedescr.itemsize)d)'],
         [True,  (4,),  'i3 = strgetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,1,'
-                       '%(strdescr.basesize)d,1)'],
+                       '%(strdescr.basesize-1)d,1)'],
         #[False, (4,),  'i3 = strgetitem(p0,i1)' '->'
-        #               'i5 = int_add(i1, %(strdescr.basesize)d);'
+        #               'i5 = int_add(i1, %(strdescr.basesize-1)d);'
         #               'i3 = gc_load_i(p0,i5,1)'],
         ## setitem str/unicode
         [True, (4,),  'i3 = strsetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,1,'
-                               '%(strdescr.basesize)d,1)'],
+                               '%(strdescr.basesize-1)d,1)'],
         [True, (2,4), 'i3 = unicodesetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,'
                                  '%(unicodedescr.itemsize)d,'
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -3,7 +3,7 @@
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
 from rpython.rlib.jit import promote, _get_virtualizable_token
-from rpython.rlib import jit_hooks, rposix
+from rpython.rlib import jit_hooks, rposix, rgc
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
 from rpython.jit.backend.detect_cpu import getcpuclass
@@ -11,7 +11,7 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 from rpython.config.config import ConfigError
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.rlib.rjitlog import rjitlog as jl
 
 
@@ -29,6 +29,7 @@
         # - floats neg and abs
         # - cast_int_to_float
         # - llexternal with macro=True
+        # - extra place for the zero after STR instances
 
         class BasicFrame(object):
             _virtualizable_ = ['i']
@@ -56,7 +57,7 @@
             return ("/home.py",0,0)
 
         jitdriver = JitDriver(greens = [],
-                              reds = ['total', 'frame', 'j'],
+                              reds = ['total', 'frame', 'prev_s', 'j'],
                               virtualizables = ['frame'],
                               get_location = get_location)
         def f(i, j):
@@ -68,9 +69,12 @@
             total = 0
             frame = Frame(i)
             j = float(j)
+            prev_s = rstr.mallocstr(16)
             while frame.i > 3:
-                jitdriver.can_enter_jit(frame=frame, total=total, j=j)
-                jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                jitdriver.can_enter_jit(frame=frame, total=total, j=j,
+                                        prev_s=prev_s)
+                jitdriver.jit_merge_point(frame=frame, total=total, j=j,
+                                          prev_s=prev_s)
                 _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
@@ -82,6 +86,11 @@
                 k = myabs1(myabs2(j))
                 if k - abs(j):  raise ValueError
                 if k - abs(-j): raise ValueError
+                s = rstr.mallocstr(16)
+                rgc.ll_write_final_null_char(s)
+                rgc.ll_write_final_null_char(prev_s)
+                if (frame.i & 3) == 0:
+                    prev_s = s
             return chr(total % 253)
         #
         class Virt2(object):
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -994,6 +994,7 @@
             basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                         self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             scale = 0
 
         self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale)
diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -2,48 +2,157 @@
 from rpython.rlib import jit
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rvmprof import cintf
+from rpython.rlib.rvmprof import cintf, vmprof_execute_code, register_code,\
+    register_code_object_class, _get_vmprof
 from rpython.jit.backend.x86.arch import WORD
 from rpython.jit.codewriter.policy import JitPolicy
 
+
 class BaseRVMProfTest(object):
-    def test_one(self):
-        py.test.skip("needs thread-locals in the JIT, which is only available "
-                     "after translation")
+
+    def setup_method(self, meth):
         visited = []
 
         def helper():
+            trace = []
             stack = cintf.vmprof_tl_stack.getraw()
-            if stack:
-                # not during tracing
-                visited.append(stack.c_value)
-            else:
-                visited.append(0)
+            while stack:
+                trace.append((stack.c_kind, stack.c_value))
+                stack = stack.c_next
+            visited.append(trace)
 
         llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
 
-        driver = jit.JitDriver(greens=[], reds='auto')
+        class CodeObj(object):
+            def __init__(self, name):
+                self.name = name
 
-        def f(n):
+        def get_code_fn(codes, code, arg, c):
+            return code
+
+        def get_name(code):
+            return "foo"
+
+        _get_vmprof().use_weaklist = False
+        register_code_object_class(CodeObj, get_name)
+
+        self.misc = visited, llfn, CodeObj, get_code_fn, get_name
+
+
+    def teardown_method(self, meth):
+        del _get_vmprof().use_weaklist
+
+
+    def test_simple(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
             i = 0
             while i < n:
-                driver.jit_merge_point()
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if code.name == "main":
+                    c = f(codes, codes[1], 1, c)
+                else:
+                    llfn()
+                    c -= 1
                 i += 1
-                llfn()
+            return c
 
-        class Hooks(jit.JitHookInterface):
-            def after_compile(self, debug_info):
-                self.raw_start = debug_info.asminfo.rawstart
-
-        hooks = Hooks()
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            return f(codes, codes[0], n, 8)
 
         null = lltype.nullptr(cintf.VMPROFSTACK)
-        cintf.vmprof_tl_stack.setraw(null)   # make it empty
-        self.meta_interp(f, [10], policy=JitPolicy(hooks))
-        v = set(visited)
-        assert 0 in v
-        v.remove(0)
-        assert len(v) == 1
-        assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
-        assert cintf.vmprof_tl_stack.getraw() == null
-        # ^^^ make sure we didn't leave anything dangling
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+
+
+    def test_leaving_with_exception(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        class MyExc(Exception):
+            def __init__(self, c):
+                self.c = c
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
+            i = 0
+            while i < n:
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if code.name == "main":
+                    try:
+                        f(codes, codes[1], 1, c)
+                    except MyExc as e:
+                        c = e.c
+                else:
+                    llfn()
+                    c -= 1
+                i += 1
+            raise MyExc(c)
+
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            try:
+                f(codes, codes[0], n, 8)
+            except MyExc as e:
+                return e.c
+
+        null = lltype.nullptr(cintf.VMPROFSTACK)
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+
+
+    def test_leaving_with_exception_in_blackhole(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        class MyExc(Exception):
+            def __init__(self, c):
+                self.c = c
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
+            i = 0
+            while True:
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if i >= n:
+                    break
+                i += 1
+                if code.name == "main":
+                    try:
+                        f(codes, codes[1], 1, c)
+                    except MyExc as e:
+                        c = e.c
+                    driver.can_enter_jit(code=code, c=c, i=i, codes=codes, n=n)
+                else:
+                    llfn()
+                    c -= 1
+            if c & 1:      # a failing guard
+                pass
+            raise MyExc(c)
+
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            try:
+                f(codes, codes[0], n, 8)
+            except MyExc as e:
+                return e.c
+
+        null = lltype.nullptr(cintf.VMPROFSTACK)
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1673,25 +1673,6 @@
         dest_addr = AddressLoc(base_loc, ofs_loc, scale, offset_loc.value)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
-    def genop_discard_strsetitem(self, op, arglocs):
-        base_loc, ofs_loc, val_loc = arglocs
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                              self.cpu.translate_support_code)
-        assert itemsize == 1
-        dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize)
-        self.mc.MOV8(dest_addr, val_loc.lowest8bits())
-
-    def genop_discard_unicodesetitem(self, op, arglocs):
-        base_loc, ofs_loc, val_loc = arglocs
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                              self.cpu.translate_support_code)
-        if itemsize == 4:
-            self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc)
-        elif itemsize == 2:
-            self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc)
-        else:
-            assert 0, itemsize
-
     # genop_discard_setfield_raw = genop_discard_setfield_gc
 
     def genop_math_read_timestamp(self, op, arglocs, resloc):
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1219,6 +1219,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                                   self.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self.assembler.load_effective_addr(ofsloc, ofs_items, scale,
                                            resloc, baseloc)
diff --git a/rpython/jit/backend/x86/test/test_rvmprof.py b/rpython/jit/backend/x86/test/test_rvmprof.py
--- a/rpython/jit/backend/x86/test/test_rvmprof.py
+++ b/rpython/jit/backend/x86/test/test_rvmprof.py
@@ -3,5 +3,5 @@
 from rpython.jit.backend.test.test_rvmprof import BaseRVMProfTest
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
 

From pypy.commits at gmail.com  Thu Aug 11 11:47:03 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 11 Aug 2016 08:47:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: scatter unroll_safe to more
 dispatch function that have a loop inside
Message-ID: <57ac9df7.87941c0a.4ba94.25f0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async
Changeset: r86145:bdc365a69c47
Date: 2016-08-11 17:17 +0200
http://bitbucket.org/pypy/pypy/changeset/bdc365a69c47/

Log:	scatter unroll_safe to more dispatch function that have a loop
	inside

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1343,6 +1343,7 @@
         raise BytecodeCorruption("unknown opcode, ofs=%d, code=%d, name=%s" %
                                  (ofs, ord(c), name) )
 
+    @jit.unroll_safe
     def BUILD_MAP(self, itemcount, next_instr):
         w_dict = self.space.newdict()
         for i in range(itemcount):
@@ -1359,6 +1360,7 @@
             self.space.call_method(w_set, 'add', w_item)
         self.pushvalue(w_set)
 
+    @jit.unroll_safe
     def BUILD_SET_UNPACK(self, itemcount, next_instr):
         space = self.space
         w_sum = space.newset()
@@ -1398,6 +1400,7 @@
         w_sum = self.list_unpack_helper(itemcount)
         self.pushvalue(w_sum)
 
+    @jit.unroll_safe
     def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr):
         space = self.space
         num_maps = itemcount & 0xff
@@ -1430,6 +1433,7 @@
             num_maps -= 1
         self.pushvalue(w_dict)
 
+    @jit.unroll_safe
     def BUILD_MAP_UNPACK(self, itemcount, next_instr):
         space = self.space
         w_dict = space.newdict()

From pypy.commits at gmail.com  Thu Aug 11 11:49:01 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 11 Aug 2016 08:49:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merged in marky1991/pypy_new/py3k (pull
 request #468)
Message-ID: <57ac9e6d.a719c20a.aba65.2c28@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86152:e2f0f6eced42
Date: 2016-08-11 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/e2f0f6eced42/

Log:	Merged in marky1991/pypy_new/py3k (pull request #468)

	Py3k: Fix Translation for FreeBSD

diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -15,8 +15,9 @@
 
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
-        includes=['unistd.h', 'sys/syscall.h'])
+        includes=['unistd.h', 'sys/syscall.h', 'sys/stat.h'])
     HAVE_SYS_SYSCALL_H = platform.Has("syscall")
+    HAVE_SYS_STAT_H = platform.Has("stat")
     HAVE_SETSID = platform.Has("setsid")
 
 config = platform.configure(CConfig)
@@ -29,6 +30,8 @@
 compile_extra = []
 if config['HAVE_SYS_SYSCALL_H']:
     compile_extra.append("-DHAVE_SYS_SYSCALL_H")
+if config['HAVE_SYS_STAT_H']:
+    compile_extra.append("-DHAVE_SYS_STAT_H")
 if config['HAVE_SETSID']:
     compile_extra.append("-DHAVE_SETSID")
 
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -159,7 +159,6 @@
         libraries=rtime.libraries
     )
     CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
@@ -233,7 +232,6 @@
 HAS_CLOCK_MONOTONIC = cConfig.CLOCK_MONOTONIC is not None
 HAS_MONOTONIC = (_WIN or _MACOSX or
                  (HAS_CLOCK_GETTIME and (HAS_CLOCK_HIGHRES or HAS_CLOCK_MONOTONIC)))
-clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
 
@@ -1030,7 +1028,10 @@
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
                 if rffi.cast(lltype.Signed, ret) != -1:
-                    cpu_time = float(tms.c_tms_utime + tms.c_tms_stime)
+                    cpu_time = float(rffi.cast(lltype.Signed,
+                                               tms.c_tms_utime) +
+                                     rffi.cast(lltype.Signed,
+                                               tms.c_tms_stime))
                     if w_info is not None:
                         _setinfo(space, w_info, "times()",
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
@@ -1038,7 +1039,7 @@
                     return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
         return clock(space)
 
-_clock = external('clock', [], clock_t)
+_clock = external('clock', [], rposix.CLOCK_T)
 def clock(space, w_info=None):
     """clock() -> floating point number
 
@@ -1052,7 +1053,7 @@
             pass
     value = _clock()
     # Is this casting correct?
-    if value == rffi.cast(clock_t, -1):
+    if intmask(value) == intmask(rffi.cast(rposix.CLOCK_T, -1)):
         raise oefmt(space.w_RuntimeError,
                     "the processor time used is not available or its value"
                     "cannot be represented")

From pypy.commits at gmail.com  Thu Aug 11 11:49:21 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 11 Aug 2016 08:49:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Working on fixing freebsd.
Message-ID: <57ac9e81.43681c0a.ff3f.2446@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k
Changeset: r86146:a7a048ffb511
Date: 2016-08-02 10:54 -0400
http://bitbucket.org/pypy/pypy/changeset/a7a048ffb511/

Log:	Working on fixing freebsd.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -159,7 +159,6 @@
         libraries=rtime.libraries
     )
     CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
@@ -233,7 +232,6 @@
 HAS_CLOCK_MONOTONIC = cConfig.CLOCK_MONOTONIC is not None
 HAS_MONOTONIC = (_WIN or _MACOSX or
                  (HAS_CLOCK_GETTIME and (HAS_CLOCK_HIGHRES or HAS_CLOCK_MONOTONIC)))
-clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
 
@@ -1032,7 +1030,7 @@
                     return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
         return clock(space)
 
-_clock = external('clock', [], clock_t)
+_clock = external('clock', [], rposix.CLOCK_T)
 def clock(space, w_info=None):
     """clock() -> floating point number
 
@@ -1046,7 +1044,7 @@
             pass
     value = _clock()
     # Is this casting correct?
-    if value == rffi.cast(clock_t, -1):
+    if value == rffi.cast(rposix.CLOCK_T, -1):
         raise oefmt(space.w_RuntimeError,
                     "the processor time used is not available or its value"
                     "cannot be represented")

From pypy.commits at gmail.com  Thu Aug 11 11:49:26 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 11 Aug 2016 08:49:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Mostly fix translation on freebsd. A
 failure still happens late in translation, but rtyping works at least.
Message-ID: <57ac9e86.d4e01c0a.58539.4911@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k
Changeset: r86148:2d01427fae77
Date: 2016-08-02 18:57 +0000
http://bitbucket.org/pypy/pypy/changeset/2d01427fae77/

Log:	Mostly fix translation on freebsd. A failure still happens late in
	translation, but rtyping works at least.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -1022,7 +1022,10 @@
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
                 if rffi.cast(lltype.Signed, ret) != -1:
-                    cpu_time = float(tms.c_tms_utime + tms.c_tms_stime)
+                    cpu_time = float(rffi.cast(lltype.Signed,
+                                               tms.c_tms_utime) +
+                                     rffi.cast(lltype.Signed,
+                                               tms.c_tms_stime))
                     if w_info is not None:
                         _setinfo(space, w_info, "times()",
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
@@ -1044,7 +1047,7 @@
             pass
     value = _clock()
     # Is this casting correct?
-    if value == rffi.cast(rposix.CLOCK_T, -1):
+    if intmask(value) == intmask(rffi.cast(rposix.CLOCK_T, -1)):
         raise oefmt(space.w_RuntimeError,
                     "the processor time used is not available or its value"
                     "cannot be represented")

From pypy.commits at gmail.com  Thu Aug 11 11:49:24 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 11 Aug 2016 08:49:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merging.
Message-ID: <57ac9e84.c310c20a.6b7b5.240f@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k
Changeset: r86147:a5d44d8c7857
Date: 2016-08-02 10:54 -0400
http://bitbucket.org/pypy/pypy/changeset/a5d44d8c7857/

Log:	Merging.

diff too long, truncating to 2000 out of 4974 lines

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -452,7 +452,7 @@
     RegrTest('test_userstring.py', core=True),
     RegrTest('test_uu.py'),
     RegrTest('test_uuid.py'),
-    RegrTest('test_venv.py'),
+    RegrTest('test_venv.py', usemodules="struct"),
     RegrTest('test_wait3.py', usemodules="thread"),
     RegrTest('test_wait4.py', usemodules="thread"),
     RegrTest('test_warnings.py', core=True),
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -211,11 +211,10 @@
 
     def test_identity_vs_id_primitives(self):
         import sys
-        l = range(-10, 10, 2)
+        l = list(range(-10, 10, 2))
         for i in [0, 1, 3]:
             l.append(float(i))
             l.append(i + 0.1)
-            l.append(long(i))
             l.append(i + sys.maxsize)
             l.append(i - sys.maxsize)
             l.append(i + 1j)
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -976,3 +976,12 @@
         raises(TypeError, "u''.encode(None)")
         raises(TypeError, "str(b'', encoding=None)")
         raises(TypeError, 'u"".encode("utf-8", None)')
+
+    def test_casefold(self):
+        assert 'hello'.casefold() == 'hello'
+        assert 'hELlo'.casefold() == 'hello'
+        assert '&#223;'.casefold() == 'ss'
+        assert '&#64257;'.casefold() == 'fi'
+        assert '\u03a3'.casefold() == '\u03c3'
+        assert 'A\u0345\u03a3'.casefold() == 'a\u03b9\u03c3'
+        assert '\u00b5'.casefold() == '\u03bc'
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -414,6 +414,19 @@
     def _join_check_item(self, space, w_obj):
         return not space.isinstance_w(w_obj, space.w_unicode)
 
+    def descr_casefold(self, space):
+        value = self._val(space)
+        builder = self._builder(len(value))
+        for c in value:
+            c_ord = ord(c)
+            folded = unicodedb.casefold_lookup(c_ord)
+            if folded is None:
+                builder.append(unichr(unicodedb.tolower(c_ord)))
+            else:
+                for r in folded:
+                    builder.append(unichr(r))
+        return self._new(builder.build())
+
     def descr_isdecimal(self, space):
         return self._is_generic(space, '_isdecimal')
 
@@ -815,6 +828,12 @@
         and there is at least one character in S, False otherwise.
         """
 
+    def casefold():
+        """S.casefold() -> str
+
+        Return a version of S suitable for caseless comparisons.
+        """
+
     def isdecimal():
         """S.isdecimal() -> bool
 
@@ -1105,6 +1124,8 @@
 
     capitalize = interp2app(W_UnicodeObject.descr_capitalize,
                             doc=UnicodeDocstrings.capitalize.__doc__),
+    casefold = interp2app(W_UnicodeObject.descr_casefold,
+                            doc=UnicodeDocstrings.casefold.__doc__),
     center = interp2app(W_UnicodeObject.descr_center,
                         doc=UnicodeDocstrings.center.__doc__),
     count = interp2app(W_UnicodeObject.descr_count,
diff --git a/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt b/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt
@@ -0,0 +1,912 @@
+# CaseFolding-3.2.0.txt
+# Date: 2002-03-22,20:54:33 GMT [MD]
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#65533;e" to match.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, see
+# UTR #21 Case Mappings, at http://www.unicode.org/unicode/reports/tr21/
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F2; C; 03C3; # GREEK LUNATE SIGMA SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N
+24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O
+24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P
+24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q
+24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R
+24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S
+24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T
+24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U
+24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V
+24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W
+24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X
+24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y
+24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z
+FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF
+FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI
+FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
+FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI
+FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL
+FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T
+FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST
+FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW
+FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH
+FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI
+FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW
+FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH
+FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A
+FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B
+FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C
+FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D
+FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E
+FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F
+FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G
+FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H
+FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I
+FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J
+FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K
+FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L
+FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M
+FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N
+FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O
+FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P
+FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q
+FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R
+FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S
+FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T
+FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U
+FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V
+FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W
+FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X
+FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y
+FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
+10400; C; 10428; # DESERET CAPITAL LETTER LONG I
+10401; C; 10429; # DESERET CAPITAL LETTER LONG E
+10402; C; 1042A; # DESERET CAPITAL LETTER LONG A
+10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH
+10404; C; 1042C; # DESERET CAPITAL LETTER LONG O
+10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO
+10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I
+10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E
+10408; C; 10430; # DESERET CAPITAL LETTER SHORT A
+10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH
+1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O
+1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO
+1040C; C; 10434; # DESERET CAPITAL LETTER AY
+1040D; C; 10435; # DESERET CAPITAL LETTER OW
+1040E; C; 10436; # DESERET CAPITAL LETTER WU
+1040F; C; 10437; # DESERET CAPITAL LETTER YEE
+10410; C; 10438; # DESERET CAPITAL LETTER H
+10411; C; 10439; # DESERET CAPITAL LETTER PEE
+10412; C; 1043A; # DESERET CAPITAL LETTER BEE
+10413; C; 1043B; # DESERET CAPITAL LETTER TEE
+10414; C; 1043C; # DESERET CAPITAL LETTER DEE
+10415; C; 1043D; # DESERET CAPITAL LETTER CHEE
+10416; C; 1043E; # DESERET CAPITAL LETTER JEE
+10417; C; 1043F; # DESERET CAPITAL LETTER KAY
+10418; C; 10440; # DESERET CAPITAL LETTER GAY
+10419; C; 10441; # DESERET CAPITAL LETTER EF
+1041A; C; 10442; # DESERET CAPITAL LETTER VEE
+1041B; C; 10443; # DESERET CAPITAL LETTER ETH
+1041C; C; 10444; # DESERET CAPITAL LETTER THEE
+1041D; C; 10445; # DESERET CAPITAL LETTER ES
+1041E; C; 10446; # DESERET CAPITAL LETTER ZEE
+1041F; C; 10447; # DESERET CAPITAL LETTER ESH
+10420; C; 10448; # DESERET CAPITAL LETTER ZHEE
+10421; C; 10449; # DESERET CAPITAL LETTER ER
+10422; C; 1044A; # DESERET CAPITAL LETTER EL
+10423; C; 1044B; # DESERET CAPITAL LETTER EM
+10424; C; 1044C; # DESERET CAPITAL LETTER EN
+10425; C; 1044D; # DESERET CAPITAL LETTER ENG
diff --git a/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt b/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt
@@ -0,0 +1,1202 @@
+# CaseFolding-5.2.0.txt
+# Date: 2009-05-28, 23:02:34 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2009 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#223;e" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding 
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard, Version 5.0.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+# @missing 0000..10FFFF; <codepoint>
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE
+023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE
+023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR
+023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
+0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP
+0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE
+0244; C; 0289; # LATIN CAPITAL LETTER U BAR
+0245; C; 028C; # LATIN CAPITAL LETTER TURNED V
+0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE
+0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE
+024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
+024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE
+024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0370; C; 0371; # GREEK CAPITAL LETTER HETA
+0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI
+0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+03F7; C; 03F8; # GREEK CAPITAL LETTER SHO
+03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL
+03FA; C; 03FB; # GREEK CAPITAL LETTER SAN
+03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
+03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
+03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
+04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK
+04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE
+0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK
+0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA
+0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA
+0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE
+051A; C; 051B; # CYRILLIC CAPITAL LETTER QA
+051C; C; 051D; # CYRILLIC CAPITAL LETTER WE
+051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA
+0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
+0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
+0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN
+10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN
+10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN
+10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON
+10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN
+10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN
+10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN
+10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN
+10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN
+10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN
+10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS
+10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN
+10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR
+10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON
+10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR
+10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR
+10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE
+10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN
+10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR
+10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN
+10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR
+10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR
+10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN
+10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR
+10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN
+10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN
+10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN
+10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL
+10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL
+10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR
+10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN
+10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN
+10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE
+10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE
+10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE
+10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE
+10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR
+10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S
+1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL
+1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V
+1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2132; C; 214E; # TURNED CAPITAL F
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N
+24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O
+24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P
+24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q
+24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R
+24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S
+24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T
+24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U
+24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V
+24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W
+24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X
+24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y
+24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z
+2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU
+2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY
+2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE
+2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI
+2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO
+2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU
+2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE
+2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO
+2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA
+2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE
+2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE
+2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I
+2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI
+2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO
+2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE
+2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE
+2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI
+2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU
+2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI
+2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI
+2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO
+2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO
+2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU
+2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU
+2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU
+2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU
+2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE
+2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA
+2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI
+2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI
+2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA
+2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU
+2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI
+2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI
+2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA
+2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU
+2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS
+2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL
+2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO
+2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS
+2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS
+2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS
+2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA
+2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA
+2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC
+2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A
+2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE
+2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR
+2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE
+2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE
+2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL
+2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER
+2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER
+2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER
+2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA
+2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK
+2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A
+2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA
+2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK
+2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H

From pypy.commits at gmail.com  Thu Aug 11 11:49:27 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 11 Aug 2016 08:49:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fixed translation for freebsd.
Message-ID: <57ac9e87.44ce1c0a.7a3bd.2997@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k
Changeset: r86149:575f3ed5760a
Date: 2016-08-02 21:40 +0000
http://bitbucket.org/pypy/pypy/changeset/575f3ed5760a/

Log:	Fixed translation for freebsd.

diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.c b/pypy/module/_posixsubprocess/_posixsubprocess.c
--- a/pypy/module/_posixsubprocess/_posixsubprocess.c
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.c
@@ -12,7 +12,7 @@
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
 #endif
-#if defined(HAVE_SYS_STAT_H) && defined(__FreeBSD__)
+#if defined(__FreeBSD__)
 #include <sys/stat.h>
 #endif
 #ifdef HAVE_SYS_SYSCALL_H

From pypy.commits at gmail.com  Thu Aug 11 11:49:31 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 11 Aug 2016 08:49:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Reapply the fix from issue 2348.
 Translation works now.
Message-ID: <57ac9e8b.497bc20a.13214.23d7@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k
Changeset: r86151:94394e83a162
Date: 2016-08-09 16:17 +0000
http://bitbucket.org/pypy/pypy/changeset/94394e83a162/

Log:	Reapply the fix from issue 2348. Translation works now.

diff --git a/rpython/translator/c/src/precommondefs.h b/rpython/translator/c/src/precommondefs.h
--- a/rpython/translator/c/src/precommondefs.h
+++ b/rpython/translator/c/src/precommondefs.h
@@ -20,7 +20,7 @@
 #define _NETBSD_SOURCE 1
 /* Define to activate features from IEEE Stds 1003.1-2001 */
 #ifndef _POSIX_C_SOURCE
-#  define _POSIX_C_SOURCE 200112L
+#  define _POSIX_C_SOURCE 200809L
 #endif
 /* Define on FreeBSD to activate all library features */
 #define __BSD_VISIBLE 1

From pypy.commits at gmail.com  Thu Aug 11 11:49:29 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 11 Aug 2016 08:49:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Reduce diff with cpython. (thanks for
 pointing this out, David\!)
Message-ID: <57ac9e89.8cc51c0a.8af16.43a4@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k
Changeset: r86150:51fa15119324
Date: 2016-08-04 11:08 -0400
http://bitbucket.org/pypy/pypy/changeset/51fa15119324/

Log:	Reduce diff with cpython. (thanks for pointing this out, David\!)

diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.c b/pypy/module/_posixsubprocess/_posixsubprocess.c
--- a/pypy/module/_posixsubprocess/_posixsubprocess.c
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.c
@@ -12,7 +12,7 @@
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
 #endif
-#if defined(__FreeBSD__)
+#if defined(HAVE_SYS_STAT_H) && defined(__FreeBSD__)
 #include <sys/stat.h>
 #endif
 #ifdef HAVE_SYS_SYSCALL_H
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -15,8 +15,9 @@
 
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
-        includes=['unistd.h', 'sys/syscall.h'])
+        includes=['unistd.h', 'sys/syscall.h', 'sys/stat.h'])
     HAVE_SYS_SYSCALL_H = platform.Has("syscall")
+    HAVE_SYS_STAT_H = platform.Has("stat")
     HAVE_SETSID = platform.Has("setsid")
 
 config = platform.configure(CConfig)
@@ -29,6 +30,8 @@
 compile_extra = []
 if config['HAVE_SYS_SYSCALL_H']:
     compile_extra.append("-DHAVE_SYS_SYSCALL_H")
+if config['HAVE_SYS_STAT_H']:
+    compile_extra.append("-DHAVE_SYS_STAT_H")
 if config['HAVE_SETSID']:
     compile_extra.append("-DHAVE_SETSID")
 

From pypy.commits at gmail.com  Thu Aug 11 12:02:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 09:02:24 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Threads: in-progress
Message-ID: <57aca190.c3881c0a.282f.49ea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86153:a852dc954edc
Date: 2016-08-11 18:01 +0200
http://bitbucket.org/pypy/pypy/changeset/a852dc954edc/

Log:	Threads: in-progress

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -591,6 +591,13 @@
     # This class is tweaked to generate one byte per _SIG_TICKER_COUNT
     # bytecodes, at the expense of not reacting to signals instantly.
 
+    # Threads: after 10'000 calls to decrement_ticker(), it should
+    # return -1.  It should also return -1 if there was a signal.
+    # This is done by calling _update_ticker_from_signals() every 100
+    # calls, and invoking rsignal.pypysig_check_and_reset(); this in
+    # turn returns -1 if there was a signal or if it was called 100
+    # times.
+
     _SIG_TICKER_COUNT = 100
     _ticker = 0
     _ticker_count = _SIG_TICKER_COUNT * 10
@@ -610,10 +617,10 @@
             if c < 0:
                 c = self._update_ticker_from_signals()
             self._ticker_count = c
-        if self.has_bytecode_counter:    # this 'if' is constant-folded
-            print ("RDBSignalActionFlag: has_bytecode_counter: "
-                   "not supported for now")
-            raise NotImplementedError
+        #if self.has_bytecode_counter:    # this 'if' is constant-folded
+        #    print ("RDBSignalActionFlag: has_bytecode_counter: "
+        #           "not supported for now")
+        #    raise NotImplementedError
         return self._ticker
 
     def _update_ticker_from_signals(self):
diff --git a/rpython/translator/c/src/signals.h b/rpython/translator/c/src/signals.h
--- a/rpython/translator/c/src/signals.h
+++ b/rpython/translator/c/src/signals.h
@@ -39,8 +39,9 @@
 
 inline static char pypysig_check_and_reset(void) {
     /* used by reverse_debugging */
-    char result = pypysig_counter.value < 0;
-    pypysig_counter.value = 0;
+    char result = --pypysig_counter.value < 0;
+    if (result)
+        pypysig_counter.value = 100;
     return result;
 }
 

From pypy.commits at gmail.com  Thu Aug 11 12:07:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 09:07:43 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Non-passing test
Message-ID: <57aca2cf.411d1c0a.db3ce.3059@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86154:1793bd4d7a98
Date: 2016-08-11 18:06 +0200
http://bitbucket.org/pypy/pypy/changeset/1793bd4d7a98/

Log:	Non-passing test

diff --git a/rpython/translator/revdb/test/test_thread.py b/rpython/translator/revdb/test/test_thread.py
--- a/rpython/translator/revdb/test/test_thread.py
+++ b/rpython/translator/revdb/test/test_thread.py
@@ -78,6 +78,64 @@
         rdb.write_call("AAAA\n")
         rdb.done()
 
+    def test_threadlocal(self):
+        class EC(object):
+            def __init__(self, value):
+                self.value = value
+        raw_thread_local = rthread.ThreadLocalReference(EC)
+
+        def bootstrap():
+            rthread.gc_thread_start()
+            _sleep(1)
+            ec = EC(4567)
+            raw_thread_local.set(ec)
+            print raw_thread_local.get().value
+            assert raw_thread_local.get() is ec
+            rthread.gc_thread_die()
+
+        def main(argv):
+            ec = EC(12)
+            raw_thread_local.set(ec)
+            rthread.start_new_thread(bootstrap, ())
+            _sleep(2)
+            print raw_thread_local.get().value
+            assert raw_thread_local.get() is ec
+            return 9
+
+        self.compile(main, backendopt=False, thread=True)
+        out = self.run('Xx')
+        # should have printed 4567 and 12
+        rdb = self.fetch_rdb([self.exename, 'Xx'])
+        th_A = rdb.main_thread_id
+        rdb.same_stack()      # RPyGilAllocate()
+        rdb.gil_release()
+
+        th_B = rdb.switch_thread()
+        assert th_B != th_A
+        b = rdb.next('!h'); assert 300 <= b < 310   # "callback": start thread
+        rdb.gil_acquire()
+        rdb.gil_release()
+
+        rdb.switch_thread(th_A)
+        rdb.same_stack()      # start_new_thread returns
+        x = rdb.next(); assert x == th_B    # result is the 'th_B' id
+        rdb.gil_acquire()
+        rdb.gil_release()
+
+        rdb.switch_thread(th_B)
+        rdb.same_stack()      # sleep() (finishes here)
+        rdb.next('i')         # sleep()
+        rdb.gil_acquire()
+        rdb.write_call("4567\n")
+        rdb.gil_release()
+
+        rdb.switch_thread(th_A)
+        rdb.same_stack()      # sleep()
+        rdb.next('i')         # sleep()
+        rdb.gil_acquire()
+        rdb.write_call("12\n")
+        rdb.done()
+
 
 class TestThreadInteractive(InteractiveTests):
     expected_stop_points = 5

From pypy.commits at gmail.com  Thu Aug 11 12:10:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 09:10:54 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fix test
Message-ID: <57aca38e.c15e1c0a.983f3.4d78@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86155:fb9456b9dfb4
Date: 2016-08-11 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/fb9456b9dfb4/

Log:	Fix test

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -366,7 +366,6 @@
         ThreadLocalReference._COUNT += 1
         ThreadLocalField.__init__(self, lltype.Signed, 'tlref%d' % unique_id,
                                   loop_invariant=loop_invariant)
-        setraw = self.setraw
         offset = self._offset
 
         def get():
@@ -383,10 +382,10 @@
         def set(value):
             assert isinstance(value, Cls) or value is None
             if we_are_translated():
-                from rpython.rtyper.annlowlevel import cast_instance_to_gcref
-                gcref = cast_instance_to_gcref(value)
-                value = lltype.cast_ptr_to_int(gcref)
-                setraw(value)
+                from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr
+                ptr = cast_instance_to_base_ptr(value)
+                _threadlocalref_seeme(self)
+                llop.threadlocalref_store(lltype.Void, offset, ptr)
                 rgc.register_custom_trace_hook(TRACETLREF, _lambda_trace_tlref)
                 rgc.ll_writebarrier(_tracetlref_obj)
             else:

From pypy.commits at gmail.com  Thu Aug 11 13:29:54 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 11 Aug 2016 10:29:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Fix and rename
 test_crap_after_starargs (allowed in PEP 448),
 dirty fix in function calls if argument order is reversed (happens if
 stararg occurs after kwarg)
Message-ID: <57acb612.411d1c0a.db3ce.4ba6@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86156:ab59452c8103
Date: 2016-08-11 19:29 +0200
http://bitbucket.org/pypy/pypy/changeset/ab59452c8103/

Log:	Fix and rename test_crap_after_starargs (allowed in PEP 448), dirty
	fix in function calls if argument order is reversed (happens if
	stararg occurs after kwarg)

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -779,9 +779,19 @@
         py.test.raises(SyntaxError, self.simple_test, "int(base=10, '2')",
                        None, None)
 
-    def test_crap_after_starargs(self):
-        source = "call(*args, *args)"
-        py.test.raises(SyntaxError, self.simple_test, source, None, None)
+    def test_starargs_after_starargs(self):
+        #allowed since PEP 448 "Additional Unpacking Generalizations"
+        source = py.code.Source("""
+        def call(*arg):
+            ret = []
+            for i in arg:
+                ret.append(i)
+            return ret
+        
+        args = [4,5,6]
+        res = call(*args, *args)
+        """)
+        self.simple_test(source, 'res', [4,5,6,4,5,6])
 
     def test_not_a_name(self):
         source = "call(a, b, c, 3=3)"
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1214,6 +1214,14 @@
                     break
                 w_value = self.popvalue()
                 w_key = self.popvalue()
+                # temporary (dirty) fix: if star-arg occurs after kwarg,
+                # arg order is reversed on stack
+                from pypy.objspace.std.listobject import W_ListObject
+                if isinstance(w_key, W_ListObject):
+                    w_key_temp = w_key
+                    w_key = w_value
+                    w_value = w_star
+                    w_star = w_key_temp
                 key = self.space.identifier_w(w_key)
                 keywords[n_keywords] = key
                 keywords_w[n_keywords] = w_value

From pypy.commits at gmail.com  Thu Aug 11 13:42:55 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 11 Aug 2016 10:42:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Fix compiler test (positional
 arguments can follow starred arguments)
Message-ID: <57acb91f.271ac20a.d0248.491d@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86157:18060b1572d4
Date: 2016-08-11 19:42 +0200
http://bitbucket.org/pypy/pypy/changeset/18060b1572d4/

Log:	Fix compiler test (positional arguments can follow starred
	arguments)

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1075,6 +1075,13 @@
                 return a, b, c
         """
         yield self.st, func, "f()", (1, [2, 3], 4)
+        func = """def f():
+            b = [4,5,6]
+            c = 7
+            a = [*b, c]
+            return a
+        """
+        yield self.st, func, "f()", [4, 5, 6, 7]
 
     def test_extended_unpacking_fail(self):
         exc = py.test.raises(SyntaxError, self.simple_test, "*a, *b = [1, 2]",
@@ -1084,9 +1091,6 @@
                              "[*b, *c] = range(10)", None, None).value
         assert exc.msg == "two starred expressions in assignment"
 
-        exc = py.test.raises(SyntaxError, self.simple_test, "a = [*b, c]",
-                             None, None).value
-        assert exc.msg == "can use starred expression only as assignment target"
         exc = py.test.raises(SyntaxError, self.simple_test, "for *a in x: pass",
                              None, None).value
         assert exc.msg == "starred assignment target must be in a list or tuple"

From pypy.commits at gmail.com  Thu Aug 11 13:58:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 10:58:04 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Hopefully correct handling of
 thread-locals
Message-ID: <57acbcac.8f8e1c0a.8df7f.6bfe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86158:0981c3747dd3
Date: 2016-08-11 19:57 +0200
http://bitbucket.org/pypy/pypy/changeset/0981c3747dd3/

Log:	Hopefully correct handling of thread-locals

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -778,6 +778,10 @@
     for field in fields:
         print >> f, ('#define RPY_TLOFS_%s  offsetof(' % field.fieldname +
                      'struct pypy_threadlocal_s, %s)' % field.fieldname)
+    if fields:
+        print >> f, '#define RPY_TLOFSFIRST  RPY_TLOFS_%s' % fields[0].fieldname
+    else:
+        print >> f, '#define RPY_TLOFSFIRST  sizeof(struct pypy_threadlocal_s)'
     print >> f, 'struct pypy_threadlocal_s {'
     print >> f, '\tint ready;'
     print >> f, '\tchar *stack_end;'
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -26,6 +26,7 @@
 #include "revdb_def.h"
 #include "src/rtyper.h"
 #include "src/mem.h"
+#include "src/threadlocal.h"
 #include "src-revdb/revdb_include.h"
 
 #define RDB_SIGNATURE   "RevDB:"
@@ -673,6 +674,7 @@
 struct replay_thread_s {
     uint64_t tid;
     stacklet_handle h;
+    struct pypy_threadlocal_s tloc;
 };
 
 static stacklet_handle replay_thread_main(stacklet_handle h, void *arg)
@@ -737,6 +739,7 @@
            can switch it away at any point later */
         struct replay_thread_main_s m;
         stacklet_handle h;
+        struct pypy_threadlocal_s *real_tloc = NULL;
         m.entry_point = entry_point;
         m.argc = argc;
         m.argv = argv;
@@ -750,6 +753,10 @@
         while (1) {
             struct replay_thread_s *node, **item, dummy;
 
+            if (real_tloc == NULL) {
+                _OP_THREADLOCALREF_ADDR_SIGHANDLER(real_tloc);
+            }
+
             if (h == NULL)
                 goto out_of_memory;
 
@@ -761,6 +768,12 @@
                     goto out_of_memory;
                 node->tid = current_thread_id;
                 node->h = h;
+                /* save the thread-locals, if any */
+                if (real_tloc != NULL)
+                    node->tloc = *real_tloc;
+                else
+                    memset(&node->tloc, 0, sizeof(node->tloc));
+
                 item = tsearch(node, &thread_tree_root, compare_replay_thread);
                 if (item == NULL)
                     goto out_of_memory;
@@ -780,6 +793,9 @@
             item = tfind(&dummy, &thread_tree_root, compare_replay_thread);
             if (item == NULL) {
                 /* it's a new thread, start it now */
+                if (real_tloc != NULL)
+                    memset(((char *)real_tloc) + RPY_TLOFSFIRST, 0,
+                           sizeof(struct pypy_threadlocal_s) - RPY_TLOFSFIRST);
                 h = stacklet_new(st_thread, replay_thread_sub, NULL);
             }
             else {
@@ -787,6 +803,8 @@
                 assert(node->tid == target_thread_id);
                 h = node->h;
                 tdelete(node, &thread_tree_root, compare_replay_thread);
+                if (real_tloc != NULL)
+                    *real_tloc = node->tloc;
                 free(node);
 
                 h = stacklet_switch(h);
@@ -1068,7 +1086,10 @@
                 target_thread_id = fetch_async_block();
                 _RPY_REVDB_PRINT("[THRD]", target_thread_id);
                 rpy_revdb.buf_limit = rpy_revdb.buf_p;
-                st_outer_controller_h = stacklet_switch(st_outer_controller_h);
+                if (target_thread_id != current_thread_id) {
+                    st_outer_controller_h = stacklet_switch(
+                                                st_outer_controller_h);
+                }
                 if (rpy_revdb.buf_limit == rpy_revdb.buf_p)
                     rpy_reverse_db_fetch(__FILE__, __LINE__);
                 return;
diff --git a/rpython/translator/revdb/test/test_thread.py b/rpython/translator/revdb/test/test_thread.py
--- a/rpython/translator/revdb/test/test_thread.py
+++ b/rpython/translator/revdb/test/test_thread.py
@@ -169,3 +169,41 @@
             child.expect(ANSWER_READY, i, Ellipsis)
         child.send(Message(CMD_FORWARD, 1))
         child.expect(ANSWER_AT_END)
+
+
+class TestThreadLocal(InteractiveTests):
+    expected_stop_points = 1
+
+    def setup_class(cls):
+        from rpython.translator.revdb.test.test_basic import compile, run
+        class EC(object):
+            def __init__(self, value):
+                self.value = value
+        raw_thread_local = rthread.ThreadLocalReference(EC)
+
+        def bootstrap():
+            rthread.gc_thread_start()
+            _sleep(1)
+            ec = EC(4567)
+            raw_thread_local.set(ec)
+            revdb.stop_point()
+            print raw_thread_local.get().value
+            assert raw_thread_local.get() is ec
+            rthread.gc_thread_die()
+
+        def main(argv):
+            ec = EC(12)
+            raw_thread_local.set(ec)
+            rthread.start_new_thread(bootstrap, ())
+            _sleep(2)
+            print raw_thread_local.get().value
+            assert raw_thread_local.get() is ec
+            return 9
+
+        compile(cls, main, backendopt=False, thread=True)
+        assert run(cls, '') == '4567\n12\n'
+
+    def test_go_threadlocal(self):
+        child = self.replay()
+        child.send(Message(CMD_FORWARD, 1))
+        child.expect(ANSWER_AT_END)

From pypy.commits at gmail.com  Thu Aug 11 14:34:07 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 11 Aug 2016 11:34:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Merge with py3.5-async
Message-ID: <57acc51f.c310c20a.6b7b5.5afb@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86159:83d383a3859c
Date: 2016-08-11 20:31 +0200
http://bitbucket.org/pypy/pypy/changeset/83d383a3859c/

Log:	Merge with py3.5-async

diff too long, truncating to 2000 out of 40779 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -85,10 +85,11 @@
     pass
 
 def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')):
+    import ast
     result = []
     for word in re_word.findall(line):
         if word.startswith('"'):
-            word = eval(word)
+            word = ast.literal_eval(word)
         result.append(word)
     return result
 
diff --git a/lib-python/2.7/test/test_hash.py b/lib-python/2.7/test/test_hash.py
--- a/lib-python/2.7/test/test_hash.py
+++ b/lib-python/2.7/test/test_hash.py
@@ -174,7 +174,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 
diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -85,7 +85,10 @@
 def_op('INPLACE_FLOOR_DIVIDE', 28)
 def_op('INPLACE_TRUE_DIVIDE', 29)
 
-def_op('STORE_MAP', 54)
+def_op('GET_AITER', 50)
+def_op('GET_ANEXT', 51)
+def_op('BEFORE_ASYNC_WITH', 52)
+
 def_op('INPLACE_ADD', 55)
 def_op('INPLACE_SUBTRACT', 56)
 def_op('INPLACE_MULTIPLY', 57)
@@ -100,11 +103,12 @@
 def_op('BINARY_OR', 66)
 def_op('INPLACE_POWER', 67)
 def_op('GET_ITER', 68)
-def_op('STORE_LOCALS', 69)
+def_op('GET_YIELD_FROM_ITER', 69)
 
 def_op('PRINT_EXPR', 70)
 def_op('LOAD_BUILD_CLASS', 71)
 def_op('YIELD_FROM', 72)
+def_op('GET_AWAITABLE', 73)
 
 def_op('INPLACE_LSHIFT', 75)
 def_op('INPLACE_RSHIFT', 76)
@@ -196,6 +200,11 @@
 def_op('SET_ADD', 146)
 def_op('MAP_ADD', 147)
 
+def_op('LOAD_CLASSDEREF', 148)
+hasfree.append(148)
+
+jrel_op('SETUP_ASYNC_WITH', 154)
+
 def_op('EXTENDED_ARG', 144)
 EXTENDED_ARG = 144
 
diff --git a/lib-python/3/test/test_hash.py b/lib-python/3/test/test_hash.py
--- a/lib-python/3/test/test_hash.py
+++ b/lib-python/3/test/test_hash.py
@@ -198,7 +198,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
     repr_ = None
diff --git a/lib-python/3/test/test_unicode.py b/lib-python/3/test/test_unicode.py
--- a/lib-python/3/test/test_unicode.py
+++ b/lib-python/3/test/test_unicode.py
@@ -2604,7 +2604,8 @@
     def test_getnewargs(self):
         text = 'abc'
         args = text.__getnewargs__()
-        self.assertIsNot(args[0], text)
+        if support.check_impl_detail():
+            self.assertIsNot(args[0], text)
         self.assertEqual(args[0], text)
         self.assertEqual(len(args), 1)
 
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -418,7 +418,7 @@
     RegrTest('test_threading.py', usemodules="thread", core=True),
     RegrTest('test_threading_local.py', usemodules="thread", core=True),
     RegrTest('test_threadsignals.py', usemodules="thread"),
-    RegrTest('test_time.py', core=True, usemodules="struct"),
+    RegrTest('test_time.py', core=True, usemodules="struct thread _rawffi"),
     RegrTest('test_timeit.py'),
     RegrTest('test_timeout.py'),
     RegrTest('test_tk.py'),
@@ -452,7 +452,7 @@
     RegrTest('test_userstring.py', core=True),
     RegrTest('test_uu.py'),
     RegrTest('test_uuid.py'),
-    RegrTest('test_venv.py'),
+    RegrTest('test_venv.py', usemodules="struct"),
     RegrTest('test_wait3.py', usemodules="thread"),
     RegrTest('test_wait4.py', usemodules="thread"),
     RegrTest('test_warnings.py', core=True),
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -198,10 +198,13 @@
     return tp._alignmentofinstances()
 
 @builtinify
-def byref(cdata):
+def byref(cdata, offset=0):
     # "pointer" is imported at the end of this module to avoid circular
     # imports
-    return pointer(cdata)
+    ptr = pointer(cdata)
+    if offset != 0:
+        ptr._buffer[0] += offset
+    return ptr
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -0,0 +1,91 @@
+# Note: uses the CFFI out-of-line ABI mode.  We can't use the API
+# mode because ffi.compile() needs to run the compiler, which
+# needs 'subprocess', which needs 'msvcrt' and '_subprocess',
+# which depend on '_pypy_winbase_cffi' already.
+#
+# Note that if you need to regenerate _pypy_winbase_cffi and
+# can't use a preexisting PyPy to do that, then running this
+# file should work as long as 'subprocess' is not imported
+# by cffi.  I had to hack in 'cffi._pycparser' to move an
+#'import subprocess' to the inside of a function.  (Also,
+# CPython+CFFI should work as well.)
+#
+# This module supports both msvcrt.py and _subprocess.py.
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pypy_winbase_cffi", None)
+
+# ---------- MSVCRT ----------
+
+ffi.cdef("""
+typedef unsigned short wint_t;
+
+int _open_osfhandle(intptr_t osfhandle, int flags);
+intptr_t _get_osfhandle(int fd);
+int _setmode(int fd, int mode);
+int _locking(int fd, int mode, long nbytes);
+
+int _kbhit(void);
+int _getch(void);
+wint_t _getwch(void);
+int _getche(void);
+wint_t _getwche(void);
+int _putch(int);
+wint_t _putwch(wchar_t);
+int _ungetch(int);
+wint_t _ungetwch(wint_t);
+""")
+
+# ---------- SUBPROCESS ----------
+
+ffi.cdef("""
+typedef struct {
+    DWORD  cb;
+    char * lpReserved;
+    char * lpDesktop;
+    char * lpTitle;
+    DWORD  dwX;
+    DWORD  dwY;
+    DWORD  dwXSize;
+    DWORD  dwYSize;
+    DWORD  dwXCountChars;
+    DWORD  dwYCountChars;
+    DWORD  dwFillAttribute;
+    DWORD  dwFlags;
+    WORD   wShowWindow;
+    WORD   cbReserved2;
+    LPBYTE lpReserved2;
+    HANDLE hStdInput;
+    HANDLE hStdOutput;
+    HANDLE hStdError;
+} STARTUPINFO, *LPSTARTUPINFO;
+
+typedef struct {
+    HANDLE hProcess;
+    HANDLE hThread;
+    DWORD  dwProcessId;
+    DWORD  dwThreadId;
+} PROCESS_INFORMATION, *LPPROCESS_INFORMATION;
+
+DWORD WINAPI GetVersion(void);
+BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD);
+BOOL WINAPI CloseHandle(HANDLE);
+HANDLE WINAPI GetCurrentProcess(void);
+BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
+                            DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
+BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
+BOOL WINAPI TerminateProcess(HANDLE, UINT);
+HANDLE WINAPI GetStdHandle(DWORD);
+""")
+
+# --------------------
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -0,0 +1,10 @@
+# auto-generated file
+import _cffi_backend
+
+ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
+    _version = 0x2601,
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+)
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -10,152 +10,99 @@
 
 # Declare external Win32 functions
 
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
+from _pypy_winbase_cffi import ffi as _ffi
+_kernel32 = _ffi.dlopen('kernel32')
 
 GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
 
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
 
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
+# Now the _subprocess module implementation
 
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
+def _WinError():
+    code, message = _ffi.getwinerror()
+    raise WindowsError(code, message)
 
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
+_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
 
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-_GetModuleFileNameW = _kernel32.GetModuleFileNameW
-_GetModuleFileNameW.argtypes = [ctypes.c_int, ctypes.c_wchar_p, ctypes.c_uint]
-_GetModuleFileNameW.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessW
-_CreateProcess.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_wchar_p, ctypes.c_wchar_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
-
-# Now the _winapi module implementation
-
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
-
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
+class _handle(object):
+    def __init__(self, c_handle):
+        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
+        self.c_handle = c_handle
+        if int(self) != -1:
+            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
 
     def __int__(self):
-        return self.handle
+        return int(_ffi.cast("intptr_t", self.c_handle))
 
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
+    def __repr__(self):
+        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
 
     def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
+        h = int(self)
+        if h != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+        return h
 
     def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
+        if int(self) != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+            _kernel32.CloseHandle(c_handle)
 
 def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
+    handles = _ffi.new("HANDLE[2]")
 
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
+    res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size)
 
     if not res:
         raise _WinError()
 
-    return _handle(read.value), _handle(write.value)
+    return _handle(handles[0]), _handle(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
+    return _handle(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
+    # CPython: the first three arguments are expected to be integers
+    target = _ffi.new("HANDLE[1]")
 
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
+    res = _kernel32.DuplicateHandle(
+        _ffi.cast("HANDLE", source_process),
+        _ffi.cast("HANDLE", source),
+        _ffi.cast("HANDLE", target_process),
+        target, access, inherit, options)
 
     if not res:
         raise _WinError()
 
-    return _handle(target.value)
+    return _handle(target[0])
+
+def _z(input):
+    if input is None:
+        return _ffi.NULL
+    if isinstance(input, basestring):
+        return str(input)
+    raise TypeError("string/unicode/None expected, got %r" % (
+        type(input).__name__,))
 
 def CreateProcess(name, command_line, process_attr, thread_attr,
                   inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
+    si = _ffi.new("STARTUPINFO *")
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
+        # CPython: these three handles are expected to be _handle objects
         if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
+            si.hStdInput = startup_info.hStdInput.c_handle
         if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
+            si.hStdOutput = startup_info.hStdOutput.c_handle
         if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
+            si.hStdError = startup_info.hStdError.c_handle
 
-    pi = _PROCESS_INFORMATION()
+    pi = _ffi.new("PROCESS_INFORMATION *")
     flags |= CREATE_UNICODE_ENVIRONMENT
 
     if env is not None:
@@ -164,47 +111,55 @@
             envbuf += "%s=%s\0" % (k, v)
         envbuf += '\0'
     else:
-        envbuf = None
+        envbuf = _ffi.NULL
 
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
+    res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL,
+                                   _ffi.NULL, inherit, flags, envbuf,
+                                   _z(start_dir), si, pi)
 
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
 
 def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
+    # CPython: the first argument is expected to be an integer.
+    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
+                                        milliseconds)
     if res < 0:
         raise _WinError()
 
     return res
 
 def GetExitCodeProcess(handle):
-    code = _c_int()
+    # CPython: the first argument is expected to be an integer.
+    code = _ffi.new("DWORD[1]")
 
-    res = _GetExitCodeProcess(int(handle), _byref(code))
+    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
 
     if not res:
         raise _WinError()
 
-    return code.value
+    return code[0]
 
 def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
+    # CPython: the first argument is expected to be an integer.
+    # The second argument is silently wrapped in a UINT.
+    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+                                     _ffi.cast("UINT", exitcode))
 
     if not res:
         raise _WinError()
 
 def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
+    stdhandle = _ffi.cast("DWORD", stdhandle)
+    res = _kernel32.GetStdHandle(stdhandle)
 
     if not res:
         return None
     else:
-        return res
+        # note: returns integer, not handle object
+        return int(_ffi.cast("intptr_t", res))
 
 def CloseHandle(handle):
     res = _CloseHandle(handle)
diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py
--- a/lib_pypy/cffi/_pycparser/__init__.py
+++ b/lib_pypy/cffi/_pycparser/__init__.py
@@ -10,7 +10,6 @@
 __all__ = ['c_lexer', 'c_parser', 'c_ast']
 __version__ = '2.14'
 
-from subprocess import Popen, PIPE
 from .c_parser import CParser
 
 
@@ -28,6 +27,7 @@
         When successful, returns the preprocessed file's contents.
         Errors from cpp will be printed out.
     """
+    from subprocess import Popen, PIPE
     path_list = [cpp_path]
     if isinstance(cpp_args, list):
         path_list += cpp_args
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -2,7 +2,7 @@
 import __pypy__
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -7,26 +7,39 @@
 # XXX incomplete: implemented only functions needed by subprocess.py
 # PAC: 2010/08 added MS locking for Whoosh
 
-import ctypes
+# 07/2016: rewrote in CFFI
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The 'msvcrt' module is only available on Windows")
+
+import _rawffi
+from _pypy_winbase_cffi import ffi as _ffi
+_lib = _ffi.dlopen(_rawffi.get_libc().name)
+
 import errno
-from ctypes_support import standard_c_lib as _c
-from ctypes_support import get_errno
-
-try:
-    open_osfhandle = _c._open_osfhandle
-except AttributeError: # we are not on windows
-    raise ImportError
 
 try: from __pypy__ import builtinify, validate_fd
 except ImportError: builtinify = validate_fd = lambda f: f
 
 
-open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int]
-open_osfhandle.restype = ctypes.c_int
+def _ioerr():
+    e = _ffi.errno
+    raise IOError(e, errno.errorcode[e])
 
-_get_osfhandle = _c._get_osfhandle
-_get_osfhandle.argtypes = [ctypes.c_int]
-_get_osfhandle.restype = ctypes.c_int
+
+ at builtinify
+def open_osfhandle(fd, flags):
+    """"open_osfhandle(handle, flags) -> file descriptor
+
+    Create a C runtime file descriptor from the file handle handle. The
+    flags parameter should be a bitwise OR of os.O_APPEND, os.O_RDONLY,
+    and os.O_TEXT. The returned file descriptor may be used as a parameter
+    to os.fdopen() to create a file object."""
+    fd = _lib._open_osfhandle(fd, flags)
+    if fd == -1:
+        _ioerr()
+    return fd
 
 @builtinify
 def get_osfhandle(fd):
@@ -38,62 +51,74 @@
         validate_fd(fd)
     except OSError as e:
         raise IOError(*e.args)
-    return _get_osfhandle(fd)
+    result = _lib._get_osfhandle(fd)
+    if result == -1:
+        _ioerr()
+    return result
 
-setmode = _c._setmode
-setmode.argtypes = [ctypes.c_int, ctypes.c_int]
-setmode.restype = ctypes.c_int
+ at builtinify
+def setmode(fd, flags):
+    """setmode(fd, mode) -> Previous mode
+
+    Set the line-end translation mode for the file descriptor fd. To set
+    it to text mode, flags should be os.O_TEXT; for binary, it should be
+    os.O_BINARY."""
+    flags = _lib._setmode(fd, flags)
+    if flags == -1:
+        _ioerr()
+    return flags
 
 LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5)
 
-_locking = _c._locking
-_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_locking.restype = ctypes.c_int
-
 @builtinify
 def locking(fd, mode, nbytes):
-    '''lock or unlock a number of bytes in a file.'''
-    rv = _locking(fd, mode, nbytes)
+    """"locking(fd, mode, nbytes) -> None
+
+    Lock part of a file based on file descriptor fd from the C runtime.
+    Raises IOError on failure. The locked region of the file extends from
+    the current file position for nbytes bytes, and may continue beyond
+    the end of the file. mode must be one of the LK_* constants listed
+    below. Multiple regions in a file may be locked at the same time, but
+    may not overlap. Adjacent regions are not merged; they must be unlocked
+    individually."""
+    rv = _lib._locking(fd, mode, nbytes)
     if rv != 0:
-        e = get_errno()
-        raise IOError(e, errno.errorcode[e])
+        _ioerr()
 
 # Console I/O routines
 
-kbhit = _c._kbhit
-kbhit.argtypes = []
-kbhit.restype = ctypes.c_int
+kbhit = _lib._kbhit
 
-getch = _c._getch
-getch.argtypes = []
-getch.restype = ctypes.c_char
+ at builtinify
+def getch():
+    return chr(_lib._getch())
 
-getwch = _c._getwch
-getwch.argtypes = []
-getwch.restype = ctypes.c_wchar
+ at builtinify
+def getwch():
+    return unichr(_lib._getwch())
 
-getche = _c._getche
-getche.argtypes = []
-getche.restype = ctypes.c_char
+ at builtinify
+def getche():
+    return chr(_lib._getche())
 
-getwche = _c._getwche
-getwche.argtypes = []
-getwche.restype = ctypes.c_wchar
+ at builtinify
+def getwche():
+    return unichr(_lib._getwche())
 
-putch = _c._putch
-putch.argtypes = [ctypes.c_char]
-putch.restype = None
+ at builtinify
+def putch(ch):
+    _lib._putch(ord(ch))
 
-putwch = _c._putwch
-putwch.argtypes = [ctypes.c_wchar]
-putwch.restype = None
+ at builtinify
+def putwch(ch):
+    _lib._putwch(ord(ch))
 
-ungetch = _c._ungetch
-ungetch.argtypes = [ctypes.c_char]
-ungetch.restype = None
+ at builtinify
+def ungetch(ch):
+    if _lib._ungetch(ord(ch)) == -1:   # EOF
+        _ioerr()
 
-ungetwch = _c._ungetwch
-ungetwch.argtypes = [ctypes.c_wchar]
-ungetwch.restype = None
-
-del ctypes
+ at builtinify
+def ungetwch(ch):
+    if _lib._ungetwch(ord(ch)) == -1:   # EOF
+        _ioerr()
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -40,7 +40,7 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
     "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy"
-    "faulthandler",
+    "faulthandler", "_jitlog",
 ])
 
 from rpython.jit.backend import detect_cpu
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -94,6 +94,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -128,9 +142,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -138,11 +149,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -171,28 +178,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
 
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
-
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -104,27 +104,24 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev liblzma-dev
-
-For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+    tk-dev libgc-dev \
+    liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel
-
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+    gdbm-devel \
+    xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses
+    libexpat-devel libffi-devel python-curses \
+    xz-devel  # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
-
 On Mac OS X, most of these build-time dependencies are installed alongside
 the Developer Tools. However, note that in order for the installation to
 find them you may need to run::
diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt
--- a/pypy/doc/config/commandline.txt
+++ b/pypy/doc/config/commandline.txt
@@ -9,7 +9,7 @@
 PyPy Python interpreter options
 -------------------------------
 
-The following options can be used after ``translate.py
+The following options can be used after ``rpython
 targetpypystandalone`` or as options to ``py.py``.
 
 .. GENERATE: objspace
@@ -22,7 +22,7 @@
 General translation options
 ---------------------------
 
-The following are options of ``translate.py``.  They must be
+The following are options of ``bin/rpython``.  They must be
 given before the ``targetxxx`` on the command line.
 
 * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]`
diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst
--- a/pypy/doc/config/index.rst
+++ b/pypy/doc/config/index.rst
@@ -15,12 +15,12 @@
 
     ./py.py <`objspace options`_>
 
-and the ``translate.py`` translation entry
+and the ``rpython/bin/rpython`` translation entry
 point which takes arguments of this form:
 
 .. parsed-literal::
 
-    ./translate.py <`translation options`_> <target>
+    ./rpython/bin/rpython <`translation options`_> <target>
 
 For the common case of ``<target>`` being ``targetpypystandalone.py``,
 you can then pass the `object space options`_ after
@@ -28,7 +28,7 @@
 
 .. parsed-literal::
 
-    ./translate.py <`translation options`_> targetpypystandalone.py <`objspace options`_>
+    ./rpython/bin/rpython <`translation options`_> targetpypystandalone.py <`objspace options`_>
 
 There is an `overview`_ of all command line arguments that can be
 passed in either position.
diff --git a/pypy/doc/config/opt.rst b/pypy/doc/config/opt.rst
--- a/pypy/doc/config/opt.rst
+++ b/pypy/doc/config/opt.rst
@@ -4,8 +4,8 @@
 This meta-option selects a default set of optimization
 settings to use during a translation.  Usage::
 
-    translate.py --opt=#
-    translate.py -O#
+    bin/rpython --opt=#
+    bin/rpython -O#
 
 where ``#`` is the desired optimization level.  The valid choices are:
 
diff --git a/pypy/doc/config/translation.dont_write_c_files.txt b/pypy/doc/config/translation.dont_write_c_files.txt
--- a/pypy/doc/config/translation.dont_write_c_files.txt
+++ b/pypy/doc/config/translation.dont_write_c_files.txt
@@ -1,4 +1,4 @@
 write the generated C files to ``/dev/null`` instead of to the disk. Useful if
-you want to use translate.py as a benchmark and don't want to access the disk.
+you want to use translation as a benchmark and don't want to access the disk.
 
 .. _`translation documentation`: ../translation.html
diff --git a/pypy/doc/config/translation.fork_before.txt b/pypy/doc/config/translation.fork_before.txt
--- a/pypy/doc/config/translation.fork_before.txt
+++ b/pypy/doc/config/translation.fork_before.txt
@@ -1,4 +1,4 @@
 This is an option mostly useful when working on the PyPy toolchain. If you use
-it, translate.py will fork before the specified phase. If the translation
+it, translation will fork before the specified phase. If the translation
 crashes after that fork, you can fix the bug in the toolchain, and continue
 translation at the fork-point.
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -122,7 +122,7 @@
     $ hg up reflex-support         # optional
 
     # This example shows python, but using pypy-c is faster and uses less memory
-    $ python rpython/translator/goal/translate.py --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy
+    $ python rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy
 
 This will build a ``pypy-c`` that includes the cppyy module, and through that,
 Reflex support.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -315,13 +315,28 @@
 
  - ``complex``
 
+ - ``str`` (empty or single-character strings only)
+
+ - ``unicode`` (empty or single-character strings only)
+
+ - ``tuple`` (empty tuples only)
+
+ - ``frozenset`` (empty frozenset only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
-Notably missing from the list above are ``str`` and ``unicode``.  If your
-code relies on comparing strings with ``is``, then it might break in PyPy.
+Note that strings of length 2 or greater can be equal without being
+identical.  Similarly, ``x is (2,)`` is not necessarily true even if
+``x`` contains a tuple and ``x == (2,)``.  The uniqueness rules apply
+only to the particular cases described above.  The ``str``, ``unicode``,
+``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a
+test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was
+equal to ``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is
+closer to CPython's, which caches precisely the empty tuple/frozenset,
+and (generally but not always) the strings and unicodes of length <= 1.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -335,3 +335,60 @@
 
 This will disable SELinux's protection and allow PyPy to configure correctly.
 Be sure to enable it again if you need it!
+
+
+How should I report a bug?
+--------------------------
+
+Our bug tracker is here: https://bitbucket.org/pypy/pypy/issues/
+
+Missing features or incompatibilities with CPython are considered
+bugs, and they are welcome.  (See also our list of `known
+incompatibilities`__.)
+
+.. __: http://pypy.org/compat.html
+
+For bugs of the kind "I'm getting a PyPy crash or a strange
+exception", please note that: **We can't do anything without
+reproducing the bug ourselves**.  We cannot do anything with
+tracebacks from gdb, or core dumps.  This is not only because the
+standard PyPy is compiled without debug symbols.  The real reason is
+that a C-level traceback is usually of no help at all in PyPy.
+Debugging PyPy can be annoying.
+
+In more details:
+
+* First, please give the exact PyPy version, and the OS.
+
+* It might help focus our search if we know if the bug can be
+  reproduced on a "``pypy --jit off``" or not.  If "``pypy --jit
+  off``" always works, then the problem might be in the JIT.
+  Otherwise, we know we can ignore that part.
+
+* If you got the bug using only Open Source components, please give a
+  step-by-step guide that we can follow to reproduce the problem
+  ourselves.  Don't assume we know anything about any program other
+  than PyPy.  We would like a guide that we can follow point by point
+  (without guessing or having to figure things out)
+  on a machine similar to yours, starting from a bare PyPy, until we
+  see the same problem.  (If you can, you can try to reduce the number
+  of steps and the time it needs to run, but that is not mandatory.)
+
+* If the bug involves Closed Source components, or just too many Open
+  Source components to install them all ourselves, then maybe you can
+  give us some temporary ssh access to a machine where the bug can be
+  reproduced.  Or, maybe we can download a VirtualBox or VMWare
+  virtual machine where the problem occurs.
+
+* If giving us access would require us to use tools other than ssh,
+  make appointments, or sign a NDA, then we can consider a commerical
+  support contract for a small sum of money.
+
+* If even that is not possible for you, then sorry, we can't help.
+
+Of course, you can try to debug the problem yourself, and we can help
+you get started if you ask on the #pypy IRC channel, but be prepared:
+debugging an annoying PyPy problem usually involves quite a lot of gdb
+in auto-generated C code, and at least some knowledge about the
+various components involved, from PyPy's own RPython source code to
+the GC and possibly the JIT.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -39,17 +39,16 @@
 library.
 
 If you want to install 3rd party libraries, the most convenient way is
-to install pip_ (unless you want to install virtualenv as explained
-below; then you can directly use pip inside virtualenvs):
+to install pip_ using ensurepip_ (unless you want to install virtualenv as 
+explained below; then you can directly use pip inside virtualenvs):
 
 .. code-block:: console
 
-    $ curl -O https://bootstrap.pypa.io/get-pip.py
-    $ ./pypy-2.1/bin/pypy get-pip.py
-    $ ./pypy-2.1/bin/pip install pygments  # for example
+    $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install pygments  # for example
 
-Third party libraries will be installed in ``pypy-2.1/site-packages``, and
-the scripts in ``pypy-2.1/bin``.
+Third party libraries will be installed in ``pypy-xxx/site-packages``, and
+the scripts in ``pypy-xxx/bin``.
 
 
 Installing using virtualenv
@@ -61,7 +60,7 @@
 checkout::
 
 	# from a tarball
-	$ virtualenv -p /opt/pypy-c-jit-41718-3fb486695f20-linux/bin/pypy my-pypy-env
+	$ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env
 
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
@@ -69,7 +68,7 @@
 Note that bin/python is now a symlink to bin/pypy.
 
 .. _pip: http://pypi.python.org/pypi/pip
-
+.. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html
 
 Building PyPy yourself
 ~~~~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -0,0 +1,41 @@
+==========
+PyPy 5.3.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.3.0, released last week,
+due to issues_ reported by users.
+
+Thanks to those who reported the issues.
+
+.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,13 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
 .. branch: fix-gen-dfa
 
 Resolves an issue with the generator script to build the dfa for Python syntax.
@@ -19,3 +26,82 @@
 .. branch: s390x-5.3-catchup
 
 Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -0,0 +1,15 @@
+===========================
+What's new in PyPy2.7 5.3.1
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3.0
+.. startrev: f4d726d1a010
+
+
+A bug-fix release, merging these changes:
+
+  * Add include guards to pymem.h, fixes issue #2321
+
+  * Make vmprof build on OpenBSD, from pull request #456
+
+  * Fix ``bytearray('').replace('a', 'ab')``, issue #2324
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -419,13 +419,16 @@
                     target_depth -= 2
                 elif (jump_op == ops.SETUP_FINALLY or
                       jump_op == ops.SETUP_EXCEPT or
-                      jump_op == ops.SETUP_WITH):
+                      jump_op == ops.SETUP_WITH or
+                      jump_op == ops.SETUP_ASYNC_WITH):
                     if jump_op == ops.SETUP_FINALLY:
                         target_depth += 4
                     elif jump_op == ops.SETUP_EXCEPT:
                         target_depth += 4
                     elif jump_op == ops.SETUP_WITH:
                         target_depth += 3
+                    elif jump_op == ops.SETUP_ASYNC_WITH:
+                        target_depth += 3
                     if target_depth > self._max_depth:
                         self._max_depth = target_depth
                 elif (jump_op == ops.JUMP_IF_TRUE_OR_POP or
@@ -640,6 +643,13 @@
     ops.LOAD_DEREF: 1,
     ops.STORE_DEREF: -1,
     ops.DELETE_DEREF: 0,
+    
+    ops.GET_AWAITABLE: 0,
+    ops.SETUP_ASYNC_WITH: 2,
+    ops.BEFORE_ASYNC_WITH: -1,
+    ops.GET_AITER: 0,
+    ops.GET_ANEXT: 1,
+    ops.GET_YIELD_FROM_ITER: 0,
 
     ops.LOAD_CONST: 1,
 
@@ -658,6 +668,8 @@
 
     # TODO 
     ops.BUILD_LIST_FROM_ARG: 1,
+    # TODO
+    ops.LOAD_CLASSDEREF: 1,
 }
 
 
diff --git a/pypy/interpreter/astcompiler/assemble.py.orig b/pypy/interpreter/astcompiler/assemble.py.orig
deleted file mode 100644
--- a/pypy/interpreter/astcompiler/assemble.py.orig
+++ /dev/null
@@ -1,765 +0,0 @@
-"""Python control flow graph generation and bytecode assembly."""
-
-import os
-from rpython.rlib import rfloat
-from rpython.rlib.objectmodel import specialize, we_are_translated
-
-from pypy.interpreter.astcompiler import ast, consts, misc, symtable
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.pycode import PyCode
-from pypy.tool import stdlib_opcode as ops
-
-
-class StackDepthComputationError(Exception):
-    pass
-
-
-class Instruction(object):
-    """Represents a single opcode."""
-
-    def __init__(self, opcode, arg=0):
-        self.opcode = opcode
-        self.arg = arg
-        self.lineno = 0
-        self.has_jump = False
-
-    def size(self):
-        """Return the size of bytes of this instruction when it is
-        encoded.
-        """
-        if self.opcode >= ops.HAVE_ARGUMENT:
-            return (6 if self.arg > 0xFFFF else 3)
-        return 1
-
-    def jump_to(self, target, absolute=False):
-        """Indicate the target this jump instruction.
-
-        The opcode must be a JUMP opcode.
-        """
-        self.jump = (target, absolute)
-        self.has_jump = True
-
-    def __repr__(self):
-        data = [ops.opname[self.opcode]]
-        template = "<%s"
-        if self.opcode >= ops.HAVE_ARGUMENT:
-            data.append(self.arg)
-            template += " %i"
-            if self.has_jump:
-                data.append(self.jump[0])
-                template += " %s"
-        template += ">"
-        return template % tuple(data)
-
-
-class Block(object):
-    """A basic control flow block.
-
-    It has one entry point and several possible exit points.  Its
-    instructions may be jumps to other blocks, or if control flow
-    reaches the end of the block, it continues to next_block.
-    """
-
-    marked = False
-    have_return = False
-    auto_inserted_return = False
-
-    def __init__(self):
-        self.instructions = []
-        self.next_block = None
-
-    def _post_order_see(self, stack, nextblock):
-        if nextblock.marked == 0:
-            nextblock.marked = 1
-            stack.append(nextblock)
-
-    def post_order(self):
-        """Return this block and its children in post order.  This means
-        that the graph of blocks is first cleaned up to ignore
-        back-edges, thus turning it into a DAG.  Then the DAG is
-        linearized.  For example:
-
-                   A --> B -\           =>     [A, D, B, C]
-                     \-> D ---> C
-        """
-        resultblocks = []
-        stack = [self]
-        self.marked = 1
-        while stack:
-            current = stack[-1]
-            if current.marked == 1:
-                current.marked = 2
-                if current.next_block is not None:
-                    self._post_order_see(stack, current.next_block)
-            else:
-                i = current.marked - 2
-                assert i >= 0
-                while i < len(current.instructions):
-                    instr = current.instructions[i]
-                    i += 1
-                    if instr.has_jump:
-                        current.marked = i + 2
-                        self._post_order_see(stack, instr.jump[0])
-                        break
-                else:
-                    resultblocks.append(current)
-                    stack.pop()
-        resultblocks.reverse()
-        return resultblocks
-
-    def code_size(self):
-        """Return the encoded size of all the instructions in this
-        block.
-        """
-        i = 0
-        for instr in self.instructions:
-            i += instr.size()
-        return i
-
-    def get_code(self):
-        """Encode the instructions in this block into bytecode."""
-        code = []
-        for instr in self.instructions:
-            opcode = instr.opcode
-            if opcode >= ops.HAVE_ARGUMENT:
-                arg = instr.arg
-                if instr.arg > 0xFFFF:
-                    ext = arg >> 16
-                    code.append(chr(ops.EXTENDED_ARG))
-                    code.append(chr(ext & 0xFF))
-                    code.append(chr(ext >> 8))
-                    arg &= 0xFFFF
-                code.append(chr(opcode))
-                code.append(chr(arg & 0xFF))
-                code.append(chr(arg >> 8))
-            else:
-                code.append(chr(opcode))
-        return ''.join(code)
-
-
-def _make_index_dict_filter(syms, flag):
-    i = 0
-    result = {}
-    for name, scope in syms.iteritems():
-        if scope == flag:
-            result[name] = i
-            i += 1
-    return result
-
-
- at specialize.argtype(0)
-def _iter_to_dict(iterable, offset=0):
-    result = {}
-    index = offset
-    for item in iterable:
-        result[item] = index
-        index += 1
-    return result
-
-
-class PythonCodeMaker(ast.ASTVisitor):
-    """Knows how to assemble a PyCode object."""
-
-    def __init__(self, space, name, first_lineno, scope, compile_info):
-        self.space = space
-        self.name = name
-        self.first_lineno = first_lineno
-        self.compile_info = compile_info
-        self.first_block = self.new_block()
-        self.use_block(self.first_block)
-        self.names = {}
-        self.var_names = _iter_to_dict(scope.varnames)
-        self.cell_vars = _make_index_dict_filter(scope.symbols,
-                                                 symtable.SCOPE_CELL)
-        self.free_vars = _iter_to_dict(scope.free_vars, len(self.cell_vars))
-        self.w_consts = space.newdict()
-        self.argcount = 0
-        self.kwonlyargcount = 0
-        self.lineno_set = False
-        self.lineno = 0
-        self.add_none_to_final_return = True
-
-    def new_block(self):
-        return Block()
-
-    def use_block(self, block):
-        """Start emitting bytecode into block."""
-        self.current_block = block
-        self.instrs = block.instructions
-
-    def use_next_block(self, block=None):
-        """Set this block as the next_block for the last and use it."""
-        if block is None:
-            block = self.new_block()
-        self.current_block.next_block = block
-        self.use_block(block)
-        return block
-
-    def is_dead_code(self):
-        """Return False if any code can be meaningfully added to the
-        current block, or True if it would be dead code."""
-        # currently only True after a RETURN_VALUE.
-        return self.current_block.have_return
-
-    def emit_op(self, op):
-        """Emit an opcode without an argument."""
-        instr = Instruction(op)
-        if not self.lineno_set:
-            instr.lineno = self.lineno
-            self.lineno_set = True
-        if not self.is_dead_code():
-            self.instrs.append(instr)
-            if op == ops.RETURN_VALUE:
-                self.current_block.have_return = True
-        return instr
-
-    def emit_op_arg(self, op, arg):
-        """Emit an opcode with an integer argument."""
-        instr = Instruction(op, arg)
-        if not self.lineno_set:
-            instr.lineno = self.lineno
-            self.lineno_set = True
-        if not self.is_dead_code():
-            self.instrs.append(instr)
-
-    def emit_op_name(self, op, container, name):
-        """Emit an opcode referencing a name."""
-        self.emit_op_arg(op, self.add_name(container, name))
-
-    def emit_jump(self, op, block_to, absolute=False):
-        """Emit a jump opcode to another block."""
-        self.emit_op(op).jump_to(block_to, absolute)
-
-    def add_name(self, container, name):
-        """Get the index of a name in container."""
-        name = self.scope.mangle(name)
-        try:
-            index = container[name]
-        except KeyError:
-            index = len(container)
-            container[name] = index
-        return index
-
-    def add_const(self, obj):
-        """Add a W_Root to the constant array and return its location."""
-        space = self.space
-        # To avoid confusing equal but separate types, we hash store the type
-        # of the constant in the dictionary.  Moreover, we have to keep the
-        # difference between -0.0 and 0.0 floats, and this recursively in
-        # tuples.
-        w_key = self._make_key(obj)
-
-        w_len = space.finditem(self.w_consts, w_key)
-        if w_len is None:
-            w_len = space.len(self.w_consts)
-            space.setitem(self.w_consts, w_key, w_len)
-        if space.int_w(w_len) == 0:
-            self.scope.doc_removable = False
-        return space.int_w(w_len)
-
-    def _make_key(self, obj):
-        # see the tests 'test_zeros_not_mixed*' in ../test/test_compiler.py
-        space = self.space
-        w_type = space.type(obj)
-        if space.is_w(w_type, space.w_float):
-            val = space.float_w(obj)
-            if val == 0.0 and rfloat.copysign(1., val) < 0:
-                w_key = space.newtuple([obj, space.w_float, space.w_None])
-            else:
-                w_key = space.newtuple([obj, space.w_float])
-        elif space.is_w(w_type, space.w_complex):
-            w_real = space.getattr(obj, space.wrap("real"))
-            w_imag = space.getattr(obj, space.wrap("imag"))
-            real = space.float_w(w_real)
-            imag = space.float_w(w_imag)
-            real_negzero = (real == 0.0 and
-                            rfloat.copysign(1., real) < 0)
-            imag_negzero = (imag == 0.0 and
-                            rfloat.copysign(1., imag) < 0)
-            if real_negzero and imag_negzero:
-                tup = [obj, space.w_complex, space.w_None, space.w_None,
-                       space.w_None]
-            elif imag_negzero:
-                tup = [obj, space.w_complex, space.w_None, space.w_None]
-            elif real_negzero:
-                tup = [obj, space.w_complex, space.w_None]
-            else:
-                tup = [obj, space.w_complex]
-            w_key = space.newtuple(tup)
-        elif space.is_w(w_type, space.w_tuple):
-            result_w = [obj, w_type]
-            for w_item in space.fixedview(obj):
-                result_w.append(self._make_key(w_item))
-            w_key = space.newtuple(result_w[:])
-        elif isinstance(obj, PyCode):
-            w_key = space.newtuple([obj, w_type, space.id(obj)])
-        else:
-            w_key = space.newtuple([obj, w_type])
-        return w_key
-
-    def load_const(self, obj):
-        index = self.add_const(obj)
-        self.emit_op_arg(ops.LOAD_CONST, index)
-
-    def update_position(self, lineno, force=False):
-        """Possibly change the lineno for the next instructions."""
-        if force or lineno > self.lineno:
-            self.lineno = lineno
-            self.lineno_set = False
-
-    def _resolve_block_targets(self, blocks):
-        """Compute the arguments of jump instructions."""
-        last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG
-        # extends the bytecode size, so it might invalidate the offsets
-        # we've already given.  Thus we have to loop until the number of
-        # extended args is stable.  Any extended jump at all is
-        # extremely rare, so performance is not too concerning.
-        while True:
-            extended_arg_count = 0
-            offset = 0
-            force_redo = False
-            # Calculate the code offset of each block.
-            for block in blocks:
-                block.offset = offset
-                offset += block.code_size()
-            for block in blocks:
-                offset = block.offset
-                for instr in block.instructions:
-                    offset += instr.size()
-                    if instr.has_jump:
-                        target, absolute = instr.jump
-                        op = instr.opcode
-                        # Optimize an unconditional jump going to another
-                        # unconditional jump.
-                        if op == ops.JUMP_ABSOLUTE or op == ops.JUMP_FORWARD:
-                            if target.instructions:
-                                target_op = target.instructions[0].opcode
-                                if target_op == ops.JUMP_ABSOLUTE:
-                                    target = target.instructions[0].jump[0]
-                                    instr.opcode = ops.JUMP_ABSOLUTE
-                                    absolute = True
-                                elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into
-                                    # just a RETURN
-                                    instr.opcode = ops.RETURN_VALUE
-                                    instr.arg = 0
-                                    instr.has_jump = False
-                                    # The size of the code changed,
-                                    # we have to trigger another pass
-                                    force_redo = True
-                                    continue
-                        if absolute:
-                            jump_arg = target.offset
-                        else:
-                            jump_arg = target.offset - offset
-                        instr.arg = jump_arg
-                        if jump_arg > 0xFFFF:
-                            extended_arg_count += 1
-            if (extended_arg_count == last_extended_arg_count and
-                not force_redo):
-                break
-            else:
-                last_extended_arg_count = extended_arg_count
-
-    def _build_consts_array(self):
-        """Turn the applevel constants dictionary into a list."""
-        w_consts = self.w_consts
-        space = self.space
-        consts_w = [space.w_None] * space.len_w(w_consts)
-        w_iter = space.iter(w_consts)
-        first = space.wrap(0)
-        while True:
-            try:
-                w_key = space.next(w_iter)
-            except OperationError as e:
-                if not e.match(space, space.w_StopIteration):
-                    raise
-                break
-            w_index = space.getitem(w_consts, w_key)
-            w_constant = space.getitem(w_key, first)
-            w_constant = misc.intern_if_common_string(space, w_constant)
-            consts_w[space.int_w(w_index)] = w_constant
-        return consts_w
-
-    def _get_code_flags(self):
-        """Get an extra flags that should be attached to the code object."""
-        raise NotImplementedError
-
-    def _stacksize(self, blocks):
-        """Compute co_stacksize."""
-        for block in blocks:
-            block.initial_depth = 0
-        # Assumes that it is sufficient to walk the blocks in 'post-order'.
-        # This means we ignore all back-edges, but apart from that, we only
-        # look into a block when all the previous blocks have been done.
-        self._max_depth = 0
-        for block in blocks:
-            depth = self._do_stack_depth_walk(block)
-            if block.auto_inserted_return and depth != 0:
-                os.write(2, "StackDepthComputationError in %s at %s:%s\n" % (
-                    self.compile_info.filename, self.name, self.first_lineno))
-                raise StackDepthComputationError   # fatal error
-        return self._max_depth
-
-    def _next_stack_depth_walk(self, nextblock, depth):
-        if depth > nextblock.initial_depth:
-            nextblock.initial_depth = depth
-
-    def _do_stack_depth_walk(self, block):
-        depth = block.initial_depth
-        for instr in block.instructions:
-            depth += _opcode_stack_effect(instr.opcode, instr.arg)
-            if depth >= self._max_depth:
-                self._max_depth = depth
-            jump_op = instr.opcode
-            if instr.has_jump:
-                target_depth = depth
-                if jump_op == ops.FOR_ITER:
-                    target_depth -= 2
-                elif (jump_op == ops.SETUP_FINALLY or
-                      jump_op == ops.SETUP_EXCEPT or
-                      jump_op == ops.SETUP_WITH):
-                    if jump_op == ops.SETUP_FINALLY:
-                        target_depth += 4
-                    elif jump_op == ops.SETUP_EXCEPT:
-                        target_depth += 4
-                    elif jump_op == ops.SETUP_WITH:
-                        target_depth += 3
-                    if target_depth > self._max_depth:
-                        self._max_depth = target_depth
-                elif (jump_op == ops.JUMP_IF_TRUE_OR_POP or
-                      jump_op == ops.JUMP_IF_FALSE_OR_POP):
-                    depth -= 1
-                self._next_stack_depth_walk(instr.jump[0], target_depth)
-                if jump_op == ops.JUMP_ABSOLUTE or jump_op == ops.JUMP_FORWARD:
-                    # Nothing more can occur.
-                    break
-            elif jump_op == ops.RETURN_VALUE or jump_op == ops.RAISE_VARARGS:
-                # Nothing more can occur.
-                break
-        else:
-            if block.next_block:
-                self._next_stack_depth_walk(block.next_block, depth)
-        return depth
-
-    def _build_lnotab(self, blocks):
-        """Build the line number table for tracebacks and tracing."""
-        current_line = self.first_lineno
-        current_off = 0
-        table = []
-        push = table.append
-        for block in blocks:
-            offset = block.offset
-            for instr in block.instructions:
-                if instr.lineno:
-                    # compute deltas
-                    line = instr.lineno - current_line
-                    if line < 0:
-                        continue
-                    addr = offset - current_off
-                    # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned
-                    # char).  Depending on when SET_LINENO instructions
-                    # are emitted this is not always true.  Consider the
-                    # code:
-                    #     a = (1,
-                    #          b)
-                    # In the bytecode stream, the assignment to "a"
-                    # occurs after the loading of "b".  This works with
-                    # the C Python compiler because it only generates a
-                    # SET_LINENO instruction for the assignment.
-                    if line or addr:
-                        while addr > 255:
-                            push(chr(255))
-                            push(chr(0))
-                            addr -= 255
-                        while line > 255:
-                            push(chr(addr))
-                            push(chr(255))
-                            line -= 255
-                            addr = 0
-                        push(chr(addr))
-                        push(chr(line))
-                        current_line = instr.lineno
-                        current_off = offset
-                offset += instr.size()
-        return ''.join(table)
-
-    def assemble(self):
-        """Build a PyCode object."""
-        # Unless it's interactive, every code object must end in a return.
-        if not self.current_block.have_return:
-            self.use_next_block()
-            if self.add_none_to_final_return:
-                self.load_const(self.space.w_None)
-            self.emit_op(ops.RETURN_VALUE)
-            self.current_block.auto_inserted_return = True
-        # Set the first lineno if it is not already explicitly set.
-        if self.first_lineno == -1:
-            if self.first_block.instructions:
-                self.first_lineno = self.first_block.instructions[0].lineno
-            else:
-                self.first_lineno = 1
-        blocks = self.first_block.post_order()
-        self._resolve_block_targets(blocks)
-        lnotab = self._build_lnotab(blocks)
-        stack_depth = self._stacksize(blocks)
-        consts_w = self._build_consts_array()
-        names = _list_from_dict(self.names)
-        var_names = _list_from_dict(self.var_names)
-        cell_names = _list_from_dict(self.cell_vars)
-        free_names = _list_from_dict(self.free_vars, len(cell_names))
-        flags = self._get_code_flags()
-        # (Only) inherit compilerflags in PyCF_MASK
-        flags |= (self.compile_info.flags & consts.PyCF_MASK)
-        bytecode = ''.join([block.get_code() for block in blocks])
-        return PyCode(self.space,
-                      self.argcount,
-                      self.kwonlyargcount,
-                      len(self.var_names),
-                      stack_depth,
-                      flags,
-                      bytecode,
-                      list(consts_w),
-                      names,
-                      var_names,
-                      self.compile_info.filename,
-                      self.name,
-                      self.first_lineno,
-                      lnotab,
-                      free_names,
-                      cell_names,
-                      self.compile_info.hidden_applevel)
-
-
-def _list_from_dict(d, offset=0):
-    result = [None] * len(d)
-    for obj, index in d.iteritems():
-        result[index - offset] = obj
-    return result
-
-
-_static_opcode_stack_effects = {
-    ops.NOP: 0,
-
-    ops.POP_TOP: -1,
-    ops.ROT_TWO: 0,
-    ops.ROT_THREE: 0,
-    ops.DUP_TOP: 1,
-    ops.DUP_TOP_TWO: 2,
-
-    ops.UNARY_POSITIVE: 0,
-    ops.UNARY_NEGATIVE: 0,
-    ops.UNARY_NOT: 0,
-    ops.UNARY_INVERT: 0,
-
-    ops.LIST_APPEND: -1,
-    ops.SET_ADD: -1,
-    ops.MAP_ADD: -2,
-<<<<<<< local
-=======
-    # XXX 
-    ops.STORE_MAP: -2,
->>>>>>> other
-
-    ops.BINARY_POWER: -1,
-    ops.BINARY_MULTIPLY: -1,
-    ops.BINARY_MODULO: -1,
-    ops.BINARY_ADD: -1,
-    ops.BINARY_SUBTRACT: -1,
-    ops.BINARY_SUBSCR: -1,
-    ops.BINARY_FLOOR_DIVIDE: -1,
-    ops.BINARY_TRUE_DIVIDE: -1,
-    ops.BINARY_MATRIX_MULTIPLY: -1,
-    ops.BINARY_LSHIFT: -1,
-    ops.BINARY_RSHIFT: -1,
-    ops.BINARY_AND: -1,
-    ops.BINARY_OR: -1,
-    ops.BINARY_XOR: -1,
-
-    ops.INPLACE_FLOOR_DIVIDE: -1,
-    ops.INPLACE_TRUE_DIVIDE: -1,
-    ops.INPLACE_ADD: -1,
-    ops.INPLACE_SUBTRACT: -1,
-    ops.INPLACE_MULTIPLY: -1,
-    ops.INPLACE_MODULO: -1,
-    ops.INPLACE_POWER: -1,
-    ops.INPLACE_MATRIX_MULTIPLY: -1,
-    ops.INPLACE_LSHIFT: -1,
-    ops.INPLACE_RSHIFT: -1,
-    ops.INPLACE_AND: -1,
-    ops.INPLACE_OR: -1,
-    ops.INPLACE_XOR: -1,
-
-    ops.STORE_SUBSCR: -3,
-    ops.DELETE_SUBSCR: -2,
-
-    ops.GET_ITER: 0,
-    ops.FOR_ITER: 1,
-    ops.BREAK_LOOP: 0,
-    ops.CONTINUE_LOOP: 0,
-    ops.SETUP_LOOP: 0,
-
-    ops.PRINT_EXPR: -1,
-
-<<<<<<< local
-    ops.WITH_CLEANUP_START: -1,
-    ops.WITH_CLEANUP_FINISH: -1,  # XXX Sometimes more
-=======
-    # TODO 
-    ops.WITH_CLEANUP: -1,
->>>>>>> other
-    ops.LOAD_BUILD_CLASS: 1,
-<<<<<<< local
-=======
-    # TODO 
-    ops.STORE_LOCALS: -1,
->>>>>>> other
-    ops.POP_BLOCK: 0,
-    ops.POP_EXCEPT: -1,
-    ops.END_FINALLY: -4,     # assume always 4: we pretend that SETUP_FINALLY
-                             # pushes 4.  In truth, it would only push 1 and
-                             # the corresponding END_FINALLY only pops 1.
-    ops.SETUP_WITH: 1,
-    ops.SETUP_FINALLY: 0,
-    ops.SETUP_EXCEPT: 0,
-
-    ops.RETURN_VALUE: -1,
-    ops.YIELD_VALUE: 0,
-    ops.YIELD_FROM: -1,
-    ops.COMPARE_OP: -1,
-
-    # TODO 
-    ops.LOOKUP_METHOD: 1,
-
-    ops.LOAD_NAME: 1,
-    ops.STORE_NAME: -1,
-    ops.DELETE_NAME: 0,
-
-    ops.LOAD_FAST: 1,
-    ops.STORE_FAST: -1,
-    ops.DELETE_FAST: 0,
-
-    ops.LOAD_ATTR: 0,
-    ops.STORE_ATTR: -2,
-    ops.DELETE_ATTR: -1,
-
-    ops.LOAD_GLOBAL: 1,
-    ops.STORE_GLOBAL: -1,
-    ops.DELETE_GLOBAL: 0,
-    ops.DELETE_DEREF: 0,
-
-    ops.LOAD_CLOSURE: 1,
-    ops.LOAD_DEREF: 1,
-    ops.STORE_DEREF: -1,
-    ops.DELETE_DEREF: 0,

From pypy.commits at gmail.com  Thu Aug 11 15:25:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 12:25:02 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: During debugging,
 print a separator line when a thread switch occurs
Message-ID: <57acd10e.d32d1c0a.a6c21.851e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86160:e2ab29983fd5
Date: 2016-08-11 21:24 +0200
http://bitbucket.org/pypy/pypy/changeset/e2ab29983fd5/

Log:	During debugging, print a separator line when a thread switch occurs

diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -37,10 +37,19 @@
     def interact(self):
         last_command = 'help'
         previous_time = None
+        previous_thread = 0
         while True:
             last_time = self.pgroup.get_current_time()
             if last_time != previous_time:
                 print
+                if self.pgroup.get_current_thread() != previous_thread:
+                    previous_thread = self.pgroup.get_current_thread()
+                    if previous_thread == 0:
+                        print ('-------------------- in the main thread '
+                               '--------------------')
+                    else:
+                        print ('-------------------- in non-main thread '
+                               '#%d --------------------' % (previous_thread,))
                 self.pgroup.update_watch_values()
                 last_time = self.pgroup.get_current_time()
             if self.print_extra_pending_info:
@@ -49,6 +58,7 @@
             if last_time != previous_time:
                 self.pgroup.show_backtrace(complete=0)
                 previous_time = last_time
+
             prompt = '(%d)$ ' % last_time
             try:
                 cmdline = raw_input(prompt).strip()
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -120,12 +120,13 @@
         return msg
 
     def expect_ready(self):
-        msg = self.expect(ANSWER_READY, Ellipsis, Ellipsis)
+        msg = self.expect(ANSWER_READY, Ellipsis, Ellipsis, Ellipsis)
         self.update_times(msg)
 
     def update_times(self, msg):
         self.current_time = msg.arg1
         self.currently_created_objects = msg.arg2
+        self.current_thread = msg.arg3
 
     def clone(self):
         """Fork this subprocess.  Returns a new ReplayProcess() that is
@@ -252,10 +253,13 @@
     def get_currently_created_objects(self):
         return self.active.currently_created_objects
 
+    def get_current_thread(self):
+        return self.active.current_thread
+
     def _check_current_time(self, time):
         assert self.get_current_time() == time
         self.active.send(Message(CMD_FORWARD, 0))
-        return self.active.expect(ANSWER_READY, time, Ellipsis)
+        return self.active.expect(ANSWER_READY, time, Ellipsis, Ellipsis)
 
     def get_max_time(self):
         return self.total_stop_points
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -663,6 +663,7 @@
 static stacklet_thread_handle st_thread;
 static stacklet_handle st_outer_controller_h;
 static uint64_t current_thread_id, target_thread_id;
+static uint64_t current_thread_num, next_thread_num;
 static void *thread_tree_root;
 
 
@@ -672,7 +673,7 @@
     char **argv;
 };
 struct replay_thread_s {
-    uint64_t tid;
+    uint64_t tid, tnum;
     stacklet_handle h;
     struct pypy_threadlocal_s tloc;
 };
@@ -754,7 +755,9 @@
             struct replay_thread_s *node, **item, dummy;
 
             if (real_tloc == NULL) {
-                _OP_THREADLOCALREF_ADDR_SIGHANDLER(real_tloc);
+                char *p;
+                _OP_THREADLOCALREF_ADDR_SIGHANDLER(p);
+                real_tloc = (struct pypy_threadlocal_s *)p;
             }
 
             if (h == NULL)
@@ -767,6 +770,7 @@
                 if (!node)
                     goto out_of_memory;
                 node->tid = current_thread_id;
+                node->tnum = current_thread_num;
                 node->h = h;
                 /* save the thread-locals, if any */
                 if (real_tloc != NULL)
@@ -793,6 +797,7 @@
             item = tfind(&dummy, &thread_tree_root, compare_replay_thread);
             if (item == NULL) {
                 /* it's a new thread, start it now */
+                current_thread_num = next_thread_num++;
                 if (real_tloc != NULL)
                     memset(((char *)real_tloc) + RPY_TLOFSFIRST, 0,
                            sizeof(struct pypy_threadlocal_s) - RPY_TLOFSFIRST);
@@ -801,6 +806,7 @@
             else {
                 node = *item;
                 assert(node->tid == target_thread_id);
+                current_thread_num = node->tnum;
                 h = node->h;
                 tdelete(node, &thread_tree_root, compare_replay_thread);
                 if (real_tloc != NULL)
@@ -957,6 +963,8 @@
         exit(1);
     }
     current_thread_id = h.main_thread_id;
+    current_thread_num = 0;
+    next_thread_num = 1;
     if (h.ptr1 != &rpy_reverse_db_stop_point ||
         h.ptr2 != &rpy_revdb) {
         fprintf(stderr,
@@ -1389,7 +1397,7 @@
             write_answer(ANSWER_READY,
                          saved_state.stop_point_seen,
                          saved_state.unique_id_seen,
-                         0);
+                         current_thread_num);
             read_sock(&cmd, sizeof(cmd));
 
             char extra[cmd.extra_size + 1];
diff --git a/rpython/translator/revdb/test/test_thread.py b/rpython/translator/revdb/test/test_thread.py
--- a/rpython/translator/revdb/test/test_thread.py
+++ b/rpython/translator/revdb/test/test_thread.py
@@ -166,13 +166,14 @@
         child = self.replay()
         for i in range(2, 6):
             child.send(Message(CMD_FORWARD, 1))
-            child.expect(ANSWER_READY, i, Ellipsis)
+            child.expect(ANSWER_READY, i, Ellipsis,
+                         (i & 1) ^ 1)    # thread number: either 0 or 1 here
         child.send(Message(CMD_FORWARD, 1))
         child.expect(ANSWER_AT_END)
 
 
 class TestThreadLocal(InteractiveTests):
-    expected_stop_points = 1
+    expected_stop_points = 2
 
     def setup_class(cls):
         from rpython.translator.revdb.test.test_basic import compile, run
@@ -192,6 +193,7 @@
             rthread.gc_thread_die()
 
         def main(argv):
+            revdb.stop_point()
             ec = EC(12)
             raw_thread_local.set(ec)
             rthread.start_new_thread(bootstrap, ())
@@ -206,4 +208,6 @@
     def test_go_threadlocal(self):
         child = self.replay()
         child.send(Message(CMD_FORWARD, 1))
+        child.expect(ANSWER_READY, 2, Ellipsis, 1)
+        child.send(Message(CMD_FORWARD, 1))
         child.expect(ANSWER_AT_END)

From pypy.commits at gmail.com  Thu Aug 11 15:55:22 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 11 Aug 2016 12:55:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix validate tests,
 ast classes "arguments",
 "classdef" and "call" don't have stararg and kwarg anymore
Message-ID: <57acd82a.2472c20a.3b6d9.7a19@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86161:14df96842b97
Date: 2016-08-11 21:54 +0200
http://bitbucket.org/pypy/pypy/changeset/14df96842b97/

Log:	Fix validate tests, ast classes "arguments", "classdef" and "call"
	don't have stararg and kwarg anymore

diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py
--- a/pypy/interpreter/astcompiler/test/test_validate.py
+++ b/pypy/interpreter/astcompiler/test/test_validate.py
@@ -38,9 +38,8 @@
         self.mod(m, "must have Load context", "eval")
 
     def _check_arguments(self, fac, check):
-        def arguments(args=None, vararg=None, varargannotation=None,
-                      kwonlyargs=None, kwarg=None, kwargannotation=None,
-                      defaults=None, kw_defaults=None):
+        def arguments(args=None, vararg=None, kwonlyargs=None,
+                      kw_defaults=None, kwarg=None, defaults=None):
             if args is None:
                 args = []
             if kwonlyargs is None:
@@ -49,20 +48,12 @@
                 defaults = []
             if kw_defaults is None:
                 kw_defaults = []
-            args = ast.arguments(args, vararg, varargannotation, kwonlyargs,
-                                 kwarg, kwargannotation, defaults, kw_defaults)
+            args = ast.arguments(args, vararg, kwonlyargs,
+                                 kw_defaults, kwarg, defaults)
             return fac(args)
         args = [ast.arg("x", ast.Name("x", ast.Store, 0, 0))]
         check(arguments(args=args), "must have Load context")
-        check(arguments(varargannotation=ast.Num(self.space.wrap(3), 0, 0)),
-              "varargannotation but no vararg")
-        check(arguments(varargannotation=ast.Name("x", ast.Store, 0, 0), vararg="x"),
-                         "must have Load context")
         check(arguments(kwonlyargs=args), "must have Load context")
-        check(arguments(kwargannotation=ast.Num(self.space.wrap(42), 0, 0)),
-                       "kwargannotation but no kwarg")
-        check(arguments(kwargannotation=ast.Name("x", ast.Store, 0, 0),
-                          kwarg="x"), "must have Load context")
         check(arguments(defaults=[ast.Num(self.space.wrap(3), 0, 0)]),
                        "more positional defaults than args")
         check(arguments(kw_defaults=[ast.Num(self.space.wrap(4), 0, 0)]),
@@ -77,7 +68,7 @@
                           "must have Load context")
 
     def test_funcdef(self):
-        a = ast.arguments([], None, None, [], None, None, [], [])
+        a = ast.arguments([], None, [], [], None, [])
         f = ast.FunctionDef("x", a, [], [], None, 0, 0)
         self.stmt(f, "empty body on FunctionDef")
         f = ast.FunctionDef("x", a, [ast.Pass(0, 0)], [ast.Name("x", ast.Store, 0, 0)],
@@ -91,8 +82,7 @@
         self._check_arguments(fac, self.stmt)
 
     def test_classdef(self):
-        def cls(bases=None, keywords=None, starargs=None, kwargs=None,
-                body=None, decorator_list=None):
+        def cls(bases=None, keywords=None, body=None, decorator_list=None):
             if bases is None:
                 bases = []
             if keywords is None:
@@ -101,16 +91,12 @@
                 body = [ast.Pass(0, 0)]
             if decorator_list is None:
                 decorator_list = []
-            return ast.ClassDef("myclass", bases, keywords, starargs,
-                                kwargs, body, decorator_list, 0, 0)
+            return ast.ClassDef("myclass", bases, keywords,
+                                body, decorator_list, 0, 0)
         self.stmt(cls(bases=[ast.Name("x", ast.Store, 0, 0)]),
                   "must have Load context")
         self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store, 0, 0))]),
                   "must have Load context")
-        self.stmt(cls(starargs=ast.Name("x", ast.Store, 0, 0)),
-                  "must have Load context")
-        self.stmt(cls(kwargs=ast.Name("x", ast.Store, 0, 0)),
-                  "must have Load context")
         self.stmt(cls(body=[]), "empty body on ClassDef")
         self.stmt(cls(body=[None]), "None disallowed")
         self.stmt(cls(decorator_list=[ast.Name("x", ast.Store, 0, 0)]),
@@ -250,7 +236,7 @@
         self.expr(u, "must have Load context")
 
     def test_lambda(self):
-        a = ast.arguments([], None, None, [], None, None, [], [])
+        a = ast.arguments([], None, [], [], None, [])
         self.expr(ast.Lambda(a, ast.Name("x", ast.Store, 0, 0), 0, 0),
                   "must have Load context")
         def fac(args):
@@ -343,20 +329,12 @@
         func = ast.Name("x", ast.Load, 0, 0)
         args = [ast.Name("y", ast.Load, 0, 0)]
         keywords = [ast.keyword("w", ast.Name("z", ast.Load, 0, 0))]
-        stararg = ast.Name("p", ast.Load, 0, 0)
-        kwarg = ast.Name("q", ast.Load, 0, 0)
-        call = ast.Call(ast.Name("x", ast.Store, 0, 0), args, keywords, stararg,
-                        kwarg, 0, 0)
+        call = ast.Call(ast.Name("x", ast.Store, 0, 0), args, keywords, 0, 0)
         self.expr(call, "must have Load context")
-        call = ast.Call(func, [None], keywords, stararg, kwarg, 0, 0)
+        call = ast.Call(func, [None], keywords, 0, 0)
         self.expr(call, "None disallowed")
         bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store, 0, 0))]
-        call = ast.Call(func, args, bad_keywords, stararg, kwarg, 0, 0)
-        self.expr(call, "must have Load context")
-        call = ast.Call(func, args, keywords, ast.Name("z", ast.Store, 0, 0), kwarg, 0, 0)
-        self.expr(call, "must have Load context")
-        call = ast.Call(func, args, keywords, stararg,
-                        ast.Name("w", ast.Store, 0, 0), 0, 0)
+        call = ast.Call(func, args, bad_keywords, 0, 0)
         self.expr(call, "must have Load context")
 
     def test_num(self):

From pypy.commits at gmail.com  Thu Aug 11 16:15:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 Aug 2016 13:15:48 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Add commands 'nthread' and
 'bthread' to navigate thread switches.
Message-ID: <57acdcf4.c997c20a.41333.7c4d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86162:9b771a2cb860
Date: 2016-08-11 22:15 +0200
http://bitbucket.org/pypy/pypy/changeset/9b771a2cb860/

Log:	Add commands 'nthread' and 'bthread' to navigate thread switches.

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -479,6 +479,7 @@
 def command_breakpoints(cmd, extra):
     space = dbstate.space
     dbstate.breakpoint_stack_id = cmd.c_arg1
+    revdb.set_thread_breakpoint(cmd.c_arg2)
     funcnames = None
     watch_progs = []
     with non_standard_code:
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -92,6 +92,9 @@
 def breakpoint(num):
     llop.revdb_breakpoint(lltype.Void, num)
 
+def set_thread_breakpoint(tnum):
+    llop.revdb_set_thread_breakpoint(lltype.Void, tnum)
+
 @specialize.argtype(0)
 def get_unique_id(x):
     """Returns the creation number of the object 'x'.  For objects created
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -583,6 +583,7 @@
     'revdb_weakref_create': LLOp(),
     'revdb_weakref_deref':  LLOp(),
     'revdb_call_destructor': LLOp(),
+    'revdb_set_thread_breakpoint': LLOp(),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -45,7 +45,7 @@
                 if self.pgroup.get_current_thread() != previous_thread:
                     previous_thread = self.pgroup.get_current_thread()
                     if previous_thread == 0:
-                        print ('-------------------- in the main thread '
+                        print ('-------------------- in main thread #0 '
                                '--------------------')
                     else:
                         print ('-------------------- in non-main thread '
@@ -147,6 +147,9 @@
         elif num == -3:
             kind = 'stoppoint'
             name = 'explicit stop'
+        elif num == -4:
+            kind = 'switchpoint'
+            name = 'thread switch'
         else:
             kind = '?????point'
             name = repr(break_at)
@@ -245,6 +248,17 @@
         finally:
             b.stack_id = 0
 
+    @contextmanager
+    def _thread_num_break(self, thread_num):
+        # add temporarily a breakpoint that hits when we enter/leave
+        # the given thread
+        b = self.pgroup.edit_breakpoints()
+        b.thread_num = thread_num
+        try:
+            yield
+        finally:
+            b.thread_num = -1
+
     def command_next(self, argument):
         """Run forward for one step, skipping calls"""
         while True:
@@ -308,7 +322,7 @@
         """Run forward until the current function finishes"""
         stack_id = self.pgroup.get_stack_id(is_parent=True)
         if stack_id == 0:
-            print 'No stack.'
+            print 'No caller.'
         else:
             with self._stack_id_break(stack_id):
                 self.command_continue('')
@@ -317,7 +331,7 @@
         """Run backward until the current function is called"""
         stack_id = self.pgroup.get_stack_id(is_parent=True)
         if stack_id == 0:
-            print 'No stack.'
+            print 'No caller.'
         else:
             with self._stack_id_break(stack_id):
                 self.command_bcontinue('')
@@ -333,6 +347,31 @@
         self.move_backward(self.pgroup.get_current_time() - 1)
     command_bc = command_bcontinue
 
+    def _cmd_thread(self, argument, cmd_continue):
+        argument = argument.lstrip('#')
+        if argument:
+            arg = int(argument)
+            if arg == self.pgroup.get_current_thread():
+                print 'Thread #%d is already the current one.' % (arg,)
+                return
+        else:
+            # use the current thread number to detect switches to any
+            # other thread (this works because revdb.c issues a
+            # breakpoint whenever there is a switch FROM or TO the
+            # thread '#arg').
+            arg = self.pgroup.get_current_thread()
+        #
+        with self._thread_num_break(arg):
+            cmd_continue('')
+
+    def command_nthread(self, argument):
+        """Run forward until thread switch (optionally to #ARG)"""
+        self._cmd_thread(argument, self.command_continue)
+
+    def command_bthread(self, argument):
+        """Run backward until thread switch (optionally to #ARG)"""
+        self._cmd_thread(argument, self.command_bcontinue)
+
     def command_print(self, argument):
         """Print an expression or execute a line of code"""
         # locate which $NUM appear used in the expression
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -31,15 +31,17 @@
         self.watchuids = {}    # {small number: [uid...]}
         self.stack_id = 0      # breaks when leaving/entering a frame from/to
                                # the frame identified by 'stack_id'
+        self.thread_num = -1   # breaks when leaving/entering the thread_num
 
     def __repr__(self):
-        return 'AllBreakpoints(%r, %r, %r, %r)' % (
+        return 'AllBreakpoints(%r, %r, %r, %r, %r)' % (
             self.num2break, self.watchvalues, self.watchuids,
-            self.stack_id)
+            self.stack_id, self.thread_num)
 
     def compare(self, other):
         if (self.num2break == other.num2break and
-            self.stack_id == other.stack_id):
+            self.stack_id == other.stack_id and
+            self.thread_num == other.thread_num):
             if self.watchvalues == other.watchvalues:
                 return 2     # completely equal
             else:
@@ -48,12 +50,14 @@
             return 0     # different
 
     def is_empty(self):
-        return len(self.num2break) == 0 and self.stack_id == 0
+        return (len(self.num2break) == 0 and self.stack_id == 0
+                                         and self.thread_num == -1)
 
     def duplicate(self):
         a = AllBreakpoints()
         a.num2break.update(self.num2break)
         a.stack_id = self.stack_id
+        a.thread_num = self.thread_num
         return a
 
 
@@ -392,8 +396,9 @@
         if cmp == 0:
             flat = [num2break.get(n, '\x00') for n in range(N)]
             arg1 = self.all_breakpoints.stack_id
+            arg2 = self.all_breakpoints.thread_num
             extra = ''.join(flat)
-            self.active.send(Message(CMD_BREAKPOINTS, arg1, extra=extra))
+            self.active.send(Message(CMD_BREAKPOINTS, arg1, arg2, extra=extra))
             self.active.expect_ready()
         else:
             assert cmp == 1
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -663,7 +663,7 @@
 static stacklet_thread_handle st_thread;
 static stacklet_handle st_outer_controller_h;
 static uint64_t current_thread_id, target_thread_id;
-static uint64_t current_thread_num, next_thread_num;
+static uint64_t current_thread_num, next_thread_num, break_thread_num;
 static void *thread_tree_root;
 
 
@@ -726,6 +726,13 @@
         return 1;
 }
 
+static void set_current_thread_num(uint64_t tnum)
+{
+    if (break_thread_num == current_thread_num || break_thread_num == tnum)
+        rpy_reverse_db_breakpoint(-4);
+    current_thread_num = tnum;
+}
+
 RPY_EXTERN
 int rpy_reverse_db_main(Signed entry_point(Signed, char**),
                         int argc, char **argv)
@@ -797,7 +804,7 @@
             item = tfind(&dummy, &thread_tree_root, compare_replay_thread);
             if (item == NULL) {
                 /* it's a new thread, start it now */
-                current_thread_num = next_thread_num++;
+                set_current_thread_num(next_thread_num++);
                 if (real_tloc != NULL)
                     memset(((char *)real_tloc) + RPY_TLOFSFIRST, 0,
                            sizeof(struct pypy_threadlocal_s) - RPY_TLOFSFIRST);
@@ -806,7 +813,7 @@
             else {
                 node = *item;
                 assert(node->tid == target_thread_id);
-                current_thread_num = node->tnum;
+                set_current_thread_num(node->tnum);
                 h = node->h;
                 tdelete(node, &thread_tree_root, compare_replay_thread);
                 if (real_tloc != NULL)
@@ -965,6 +972,7 @@
     current_thread_id = h.main_thread_id;
     current_thread_num = 0;
     next_thread_num = 1;
+    break_thread_num = (uint64_t)-1;
     if (h.ptr1 != &rpy_reverse_db_stop_point ||
         h.ptr2 != &rpy_revdb) {
         fprintf(stderr,
@@ -1713,6 +1721,12 @@
     exit(1);
 }
 
+RPY_EXTERN
+void rpy_reverse_db_set_thread_breakpoint(int64_t tnum)
+{
+    break_thread_num = (uint64_t)tnum;
+}
+
 
 /* ------------------------------------------------------------ */
 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -227,6 +227,9 @@
    we'll just return the UID. */
 #define RPY_REVDB_CAST_PTR_TO_INT(obj)   (((struct pypy_header0 *)obj)->h_uid)
 
+#define OP_REVDB_SET_THREAD_BREAKPOINT(tnum, r)                         \
+    rpy_reverse_db_set_thread_breakpoint(tnum)
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);
@@ -249,5 +252,6 @@
 RPY_EXTERN void rpy_reverse_db_callback_loc(int);
 RPY_EXTERN void rpy_reverse_db_lock_acquire(bool_t lock_contention);
 RPY_EXTERN void rpy_reverse_db_bad_acquire_gil(void);
+RPY_EXTERN void rpy_reverse_db_set_thread_breakpoint(int64_t tnum);
 
 /* ------------------------------------------------------------ */

From pypy.commits at gmail.com  Fri Aug 12 04:36:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 01:36:42 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Some tests for prints,
 including printing $0
Message-ID: <57ad8a9a.cb7f1c0a.dff6f.75ca@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86163:d7c4d2ccc68a
Date: 2016-08-12 10:20 +0200
http://bitbucket.org/pypy/pypy/changeset/d7c4d2ccc68a/

Log:	Some tests for prints, including printing $0

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -41,7 +41,6 @@
     """
     assert space.config.translation.reverse_debugger
     dbstate.space = space
-    dbstate.w_future = space.w_Ellipsis    # a random prebuilt object
 
     make_sure_not_resized(dbstate.watch_progs)
     make_sure_not_resized(dbstate.metavars)
@@ -228,6 +227,9 @@
     revdb.stop_point(place)
 
 
+def future_object(space):
+    return space.w_Ellipsis    # a random prebuilt object
+
 def load_metavar(index):
     assert index >= 0
     space = dbstate.space
@@ -236,7 +238,7 @@
     if w_var is None:
         raise oefmt(space.w_NameError, "no constant object '$%d'",
                     index)
-    if w_var is dbstate.w_future:
+    if w_var is future_object(space):
         raise oefmt(space.w_RuntimeError,
                     "'$%d' refers to an object created later in time",
                     index)
@@ -543,7 +545,7 @@
     except KeyError:
         # uid not found, probably a future object
         dbstate.watch_futures[uid] = index_metavar
-        w_obj = dbstate.w_future
+        w_obj = future_object(space)
     set_metavar(index_metavar, w_obj)
 lambda_attachid = lambda: command_attachid
 
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1536,6 +1536,7 @@
     save_state();
     if (rpy_revdb_commands.rp_alloc) {
         protect_potential_io();
+        /* invoke the "ALLOCATING" callback from RPython */
         rpy_revdb_commands.rp_alloc(uid, new_object);
         unprotect_potential_io();
     }
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -1,12 +1,23 @@
-import py
+import py, sys
+from cStringIO import StringIO
 from rpython.rlib import revdb
-from rpython.rlib.debug import debug_print
+from rpython.rlib.debug import debug_print, ll_assert
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from rpython.translator.revdb.message import *
 from rpython.translator.revdb.process import ReplayProcessGroup, Breakpoint
 
 from hypothesis import given, strategies
 
 
+class stdout_capture(object):
+    def __enter__(self):
+        self.old_stdout = sys.stdout
+        sys.stdout = self.buffer = StringIO()
+        return self.buffer
+    def __exit__(self, *args):
+        sys.stdout = self.old_stdout
+
+
 class TestReplayProcessGroup:
 
     def setup_class(cls):
@@ -17,6 +28,10 @@
 
         class DBState:
             break_loop = -2
+            stuff = None
+            metavar = None
+            printed_stuff = None
+            watch_future = -1
         dbstate = DBState()
 
         def blip(cmd, extra):
@@ -27,8 +42,46 @@
             revdb.send_answer(42, cmd.c_cmd, -43, -44, extra)
         lambda_blip = lambda: blip
 
+        def command_print(cmd, extra):
+            if extra == 'print-me':
+                stuff = dbstate.stuff
+            elif extra == '$0':
+                stuff = dbstate.metavar
+            else:
+                assert False
+            uid = revdb.get_unique_id(stuff)
+            ll_assert(uid > 0, "uid == 0")
+            revdb.send_nextnid(uid)   # outputs '$NUM = '
+            revdb.send_output('stuff\n')
+            dbstate.printed_stuff = stuff
+        lambda_print = lambda: command_print
+
+        def command_attachid(cmd, extra):
+            index_metavar = cmd.c_arg1
+            uid = cmd.c_arg2
+            ll_assert(index_metavar == 0, "index_metavar != 0")  # in this test
+            dbstate.metavar = dbstate.printed_stuff
+            if dbstate.metavar is None:
+                # uid not found, probably a future object
+                dbstate.watch_future = uid
+        lambda_attachid = lambda: command_attachid
+
+        def command_allocating(uid, gcref):
+            stuff = cast_gcref_to_instance(Stuff, gcref)
+            # 'stuff' is just allocated; 'stuff.x' is not yet initialized
+            dbstate.printed_stuff = stuff
+            if dbstate.watch_future != -1:
+                ll_assert(dbstate.watch_future == uid,
+                          "watch_future out of sync")
+                dbstate.watch_future = -1
+                dbstate.metavar = stuff
+        lambda_allocating = lambda: command_allocating
+
         def main(argv):
             revdb.register_debug_command(100, lambda_blip)
+            revdb.register_debug_command(CMD_PRINT, lambda_print)
+            revdb.register_debug_command(CMD_ATTACHID, lambda_attachid)
+            revdb.register_debug_command("ALLOCATING", lambda_allocating)
             for i, op in enumerate(argv[1:]):
                 dbstate.stuff = Stuff()
                 dbstate.stuff.x = i + 1000
@@ -87,3 +140,26 @@
         group.active.expect(42, 100, -43, -44, 'set-breakpoint')
         group.active.expect(ANSWER_READY, 1, Ellipsis)
         group.go_forward(10, 'i')    # does not raise Breakpoint
+
+    def test_print_cmd(self):
+        group = ReplayProcessGroup(str(self.exename), self.rdbname)
+        group.go_forward(1)
+        assert group.get_current_time() == 2
+        with stdout_capture() as buf:
+            group.print_cmd('print-me')
+        assert buf.getvalue() == "$0 = stuff\n"
+        return group
+
+    def test_print_metavar(self):
+        group = self.test_print_cmd()
+        with stdout_capture() as buf:
+            group.print_cmd('$0', nids=[0])
+        assert buf.getvalue() == "$0 = stuff\n"
+
+    def test_jump_and_print_metavar(self):
+        group = self.test_print_cmd()
+        assert group.is_tainted()
+        group.jump_in_time(2)
+        with stdout_capture() as buf:
+            group.print_cmd('$0', nids=[0])
+        assert buf.getvalue() == "$0 = stuff\n"

From pypy.commits at gmail.com  Fri Aug 12 04:36:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 01:36:44 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Failing test about setting up
 a watchpoint: it doesn't force the object
Message-ID: <57ad8a9c.a710c20a.cc582.3c4b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86164:66ba9208c0e9
Date: 2016-08-12 10:28 +0200
http://bitbucket.org/pypy/pypy/changeset/66ba9208c0e9/

Log:	Failing test about setting up a watchpoint: it doesn't force the
	object to be attached

diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -77,11 +77,28 @@
                 dbstate.metavar = stuff
         lambda_allocating = lambda: command_allocating
 
+        def command_compilewatch(cmd, expression):
+            revdb.send_watch("marshalled_code", ok_flag=1)
+        lambda_compilewatch = lambda: command_compilewatch
+
+        def command_checkwatch(cmd, marshalled_code):
+            assert marshalled_code == "marshalled_code"
+            # check that $0 exists
+            if dbstate.metavar is not None:
+                revdb.send_watch("ok, stuff exists\n", ok_flag=1)
+            else:
+                revdb.send_watch("stuff does not exist!\n", ok_flag=0)
+        lambda_checkwatch = lambda: command_checkwatch
+
         def main(argv):
             revdb.register_debug_command(100, lambda_blip)
             revdb.register_debug_command(CMD_PRINT, lambda_print)
             revdb.register_debug_command(CMD_ATTACHID, lambda_attachid)
             revdb.register_debug_command("ALLOCATING", lambda_allocating)
+            revdb.register_debug_command(revdb.CMD_COMPILEWATCH,
+                                         lambda_compilewatch)
+            revdb.register_debug_command(revdb.CMD_CHECKWATCH,
+                                         lambda_checkwatch)
             for i, op in enumerate(argv[1:]):
                 dbstate.stuff = Stuff()
                 dbstate.stuff.x = i + 1000
@@ -150,16 +167,35 @@
         assert buf.getvalue() == "$0 = stuff\n"
         return group
 
-    def test_print_metavar(self):
-        group = self.test_print_cmd()
+    def _print_metavar(self, group):
         with stdout_capture() as buf:
             group.print_cmd('$0', nids=[0])
         assert buf.getvalue() == "$0 = stuff\n"
 
+    def test_print_metavar(self):
+        group = self.test_print_cmd()
+        self._print_metavar(group)
+
     def test_jump_and_print_metavar(self):
         group = self.test_print_cmd()
         assert group.is_tainted()
         group.jump_in_time(2)
-        with stdout_capture() as buf:
-            group.print_cmd('$0', nids=[0])
-        assert buf.getvalue() == "$0 = stuff\n"
+        self._print_metavar(group)
+
+    def _check_watchpoint_expr(self, group, must_exist):
+        ok_flag, compiled_code = group.compile_watchpoint_expr("$0")
+        assert ok_flag == 1
+        assert compiled_code == "marshalled_code"
+        nids = [0]
+        ok_flag, text = group.check_watchpoint_expr(compiled_code, nids)
+        print text
+        assert ok_flag == must_exist
+
+    def test_check_watchpoint_expr(self):
+        group = self.test_print_cmd()
+        self._check_watchpoint_expr(group, must_exist=1)
+
+    def test_jump_and_check_watchpoint_expr(self):
+        group = self.test_print_cmd()
+        group.jump_in_time(2)
+        self._check_watchpoint_expr(group, must_exist=1)

From pypy.commits at gmail.com  Fri Aug 12 04:36:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 01:36:45 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fix
Message-ID: <57ad8a9d.c15e1c0a.435bd.790c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86165:ff0f492637a5
Date: 2016-08-12 10:36 +0200
http://bitbucket.org/pypy/pypy/changeset/ff0f492637a5/

Log:	Fix

diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -447,6 +447,7 @@
 
     def check_watchpoint_expr(self, compiled_code, nids=None):
         if nids:
+            self.ensure_nids_to_uids(nids)
             uids = self.nids_to_uids(nids)
             self.attach_printed_objects(uids, watch_env=True)
         self.active.send(Message(CMD_CHECKWATCH, extra=compiled_code))
@@ -542,6 +543,16 @@
             uids.append(uid)
         return uids
 
+    def ensure_nids_to_uids(self, nids):
+        # Take the objects listed in nids which are alive at the
+        # current time, and return a list of uids of them.  This
+        # might require some replaying.
+        uids = []
+        if nids:
+            uids = self.nids_to_uids(nids, skip_futures=True)
+            self.ensure_printed_objects(uids)
+        return uids
+
     def attach_printed_objects(self, uids, watch_env):
         for uid in uids:
             nid = self.all_printed_objects[uid]
@@ -559,11 +570,7 @@
     def print_cmd(self, expression, nids=[]):
         """Print an expression.
         """
-        uids = []
-        if nids:
-            uids = self.nids_to_uids(nids, skip_futures=True)
-            self.ensure_printed_objects(uids)
-        #
+        uids = self.ensure_nids_to_uids(nids)
         self.active.tainted = True
         self.attach_printed_objects(uids, watch_env=False)
         self.active.send(Message(CMD_PRINT, extra=expression))

From pypy.commits at gmail.com  Fri Aug 12 05:03:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 02:03:07 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Update printed text
Message-ID: <57ad90cb.c75dc20a.853ec.4a71@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86166:4a605deff6e0
Date: 2016-08-12 11:02 +0200
http://bitbucket.org/pypy/pypy/changeset/4a605deff6e0/

Log:	Update printed text

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -978,12 +978,18 @@
         fprintf(stderr,
                 "\n"
                 "In the replaying process, the addresses are different than\n"
-                "in the recording process.  We don't support this case for\n"
-                "now, sorry.  On Linux, check if Address Space Layout\n"
-                "Randomization (ASLR) is enabled, and disable it with:\n"
+                "in the recording process.  Make sure that the executable\n"
+                "\n"
+                "    %s\n"
+                "\n"
+                "is the same one as the one that was used during recording.\n"
+                "If it is, then you may be hitting an issue with Address\n"
+                "Space Layout Randomization.  On Linux, ASLR should be\n"
+                "automatically disabled, but just in case, the following\n"
+                "command disables it manually:\n"
                 "\n"
                 "    echo 0 | sudo tee /proc/sys/kernel/randomize_va_space\n"
-                "\n");
+                "\n", argv[0]);
         exit(1);
     }
     *argc_p = h.argc;
@@ -1129,7 +1135,7 @@
         fprintf(stderr, "%s:%d: Attempted to do I/O or access raw memory\n",
                 file, line);
         if (flag_io_disabled != FID_POTENTIAL_IO) {
-            fprintf(stderr, "but we are not in a jmpbuf_protected section\n");
+            fprintf(stderr, "but we are not in a protected section\n");
             exit(1);
         }
         write_answer(ANSWER_ATTEMPT_IO, 0, 0, 0);

From pypy.commits at gmail.com  Fri Aug 12 05:35:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 02:35:28 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: When entering debugger
 commands, make floating-point results
Message-ID: <57ad9860.68adc20a.46caf.4dbd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86167:c46e5f55f170
Date: 2016-08-12 11:34 +0200
http://bitbucket.org/pypy/pypy/changeset/c46e5f55f170/

Log:	When entering debugger commands, make floating-point results
	approximately work

diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py
--- a/rpython/rlib/rdtoa.py
+++ b/rpython/rlib/rdtoa.py
@@ -3,7 +3,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator import cdir
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib import jit
+from rpython.rlib import jit, revdb
 from rpython.rlib.rstring import StringBuilder
 import py, sys
 
@@ -54,6 +54,8 @@
 def strtod(input):
     if len(input) > _INT_LIMIT:
         raise MemoryError
+    if revdb.flag_io_disabled():
+        return revdb.emulate_strtod(input)
     end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
     try:
         ll_input = rffi.str2charp(input)
@@ -236,6 +238,8 @@
          special_strings=lower_special_strings, upper=False):
     if precision > _INT_LIMIT:
         raise MemoryError
+    if revdb.flag_io_disabled():
+        return revdb.emulate_dtoa(value)
     decpt_ptr = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
     try:
         sign_ptr = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -7,7 +7,7 @@
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.annlowlevel import llhelper, hlstr
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 
 CMD_PRINT       = 1
@@ -81,6 +81,14 @@
     """
     return llop.revdb_get_value(lltype.Signed, 'p')
 
+def flag_io_disabled():
+    """Returns True if we're in the debugger typing commands."""
+    if we_are_translated():
+        if fetch_translated_config().translation.reverse_debugger:
+            flag = llop.revdb_get_value(lltype.Signed, 'i')
+            return flag != ord('R')  # FID_REGULAR_MODE
+    return False
+
 ## @specialize.arg(1)
 ## def go_forward(time_delta, callback):
 ##     """For RPython debug commands: tells that after this function finishes,
@@ -203,3 +211,22 @@
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+
+
+# ____________________________________________________________
+
+# Emulation for strtod() and dtoa() when running debugger commands
+# (we can't easily just call C code there).  The emulation can return
+# a crude result.  Hack hack hack.
+
+_INVALID_STRTOD = -3.46739514239368e+113
+
+def emulate_strtod(input):
+    d = llop.revdb_strtod(lltype.Float, input)
+    if d == _INVALID_STRTOD:
+        raise ValueError
+    return d
+
+def emulate_dtoa(value):
+    s = llop.revdb_dtoa(lltype.Ptr(rstr.STR), value)
+    return hlstr(s)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -584,6 +584,8 @@
     'revdb_weakref_deref':  LLOp(),
     'revdb_call_destructor': LLOp(),
     'revdb_set_thread_breakpoint': LLOp(),
+    'revdb_strtod':         LLOp(sideeffects=False),
+    'revdb_dtoa':           LLOp(sideeffects=False),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1523,6 +1523,8 @@
                 saved_state.unique_id_seen);
     case 'p':       /* current_place() */
         return current_place;
+    case 'i':       /* flag_io_disabled() */
+        return flag_io_disabled;
     default:
         return -1;
     }
@@ -1734,6 +1736,39 @@
     break_thread_num = (uint64_t)tnum;
 }
 
+#define INVALID_STRTOD  (-3.46739514239368e+113)
+
+RPY_EXTERN
+double rpy_reverse_db_strtod(RPyString *s)
+{
+    /* approximate hacks only */
+    double result;
+    char *endptr = NULL;
+    char buffer[8192];
+    size_t size = RPyString_Size(s);
+
+    if (size >= sizeof(buffer))
+        return INVALID_STRTOD;
+    memcpy(buffer, _RPyString_AsString(s), size);
+    buffer[size] = '\0';
+    result = strtod(buffer, &endptr);
+    if (endptr == NULL || *endptr != '\0')
+        return INVALID_STRTOD;
+    return result;
+}
+
+RPY_EXTERN RPyString *rpy_reverse_db_dtoa(double d)
+{
+    char buffer[128];
+    RPyString *result;
+    int size;
+    size = snprintf(buffer, sizeof(buffer), "%g", d);
+    if (size < 0) size = 0;   /* XXX? */
+    result = make_rpy_string(size);
+    memcpy(_RPyString_AsString(result), buffer, size);
+    return result;
+}
+
 
 /* ------------------------------------------------------------ */
 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -230,6 +230,12 @@
 #define OP_REVDB_SET_THREAD_BREAKPOINT(tnum, r)                         \
     rpy_reverse_db_set_thread_breakpoint(tnum)
 
+#define OP_REVDB_STRTOD(s, r)                                           \
+    r = rpy_reverse_db_strtod(s)
+
+#define OP_REVDB_DTOA(d, r)                                             \
+    r = rpy_reverse_db_dtoa(d)
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);
@@ -253,5 +259,7 @@
 RPY_EXTERN void rpy_reverse_db_lock_acquire(bool_t lock_contention);
 RPY_EXTERN void rpy_reverse_db_bad_acquire_gil(void);
 RPY_EXTERN void rpy_reverse_db_set_thread_breakpoint(int64_t tnum);
+RPY_EXTERN double rpy_reverse_db_strtod(RPyString *s);
+RPY_EXTERN RPyString *rpy_reverse_db_dtoa(double d);
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -1,6 +1,6 @@
 import py, sys
 from cStringIO import StringIO
-from rpython.rlib import revdb
+from rpython.rlib import revdb, rdtoa
 from rpython.rlib.debug import debug_print, ll_assert
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from rpython.translator.revdb.message import *
@@ -47,6 +47,10 @@
                 stuff = dbstate.stuff
             elif extra == '$0':
                 stuff = dbstate.metavar
+            elif extra == '2.35':
+                val = rdtoa.strtod('2.35')
+                revdb.send_output(rdtoa.dtoa(val))
+                return
             else:
                 assert False
             uid = revdb.get_unique_id(stuff)
@@ -199,3 +203,9 @@
         group = self.test_print_cmd()
         group.jump_in_time(2)
         self._check_watchpoint_expr(group, must_exist=1)
+
+    def test_rdtoa(self):
+        group = ReplayProcessGroup(str(self.exename), self.rdbname)
+        with stdout_capture() as buf:
+            group.print_cmd('2.35')
+        assert buf.getvalue() == "2.35"

From pypy.commits at gmail.com  Fri Aug 12 05:54:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 02:54:06 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: translation fix
Message-ID: <57ad9cbe.c2a5c20a.de387.55eb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86168:bec055ffd7c0
Date: 2016-08-12 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/bec055ffd7c0/

Log:	translation fix

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -229,4 +229,6 @@
 
 def emulate_dtoa(value):
     s = llop.revdb_dtoa(lltype.Ptr(rstr.STR), value)
-    return hlstr(s)
+    s = hlstr(s)
+    assert s is not None
+    return s

From pypy.commits at gmail.com  Fri Aug 12 06:27:28 2016
From: pypy.commits at gmail.com (vext01)
Date: Fri, 12 Aug 2016 03:27:28 -0700 (PDT)
Subject: [pypy-commit] pypy refactor_rmmap: Refactor rmmap.py JIT support
 into its own file.
Message-ID: <57ada490.c41f1c0a.9b2b9.a4aa@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: refactor_rmmap
Changeset: r86169:a80e988edfd0
Date: 2016-08-12 11:26 +0100
http://bitbucket.org/pypy/pypy/changeset/a80e988edfd0/

Log:	Refactor rmmap.py JIT support into its own file.

	And test with a W^X patch to the build system.

diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -1,7 +1,7 @@
 import sys
 from rpython.rlib.rarithmetic import intmask, r_uint, LONG_BIT
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rmmap
+from rpython.jit.backend.llsupport import rmmap
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
 from rpython.rtyper.lltypesystem import lltype, rffi
diff --git a/rpython/jit/backend/llsupport/rmmap.py b/rpython/jit/backend/llsupport/rmmap.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/rmmap.py
@@ -0,0 +1,145 @@
+"""mmap for the JIT
+
+Derived from rlib.rmmap
+"""
+
+# borrow a few bits from our rlib cousin, but we must not share functions
+from rpython.rlib.rmmap import _POSIX, _MS_WINDOWS, _CYGWIN, constants, CConfig
+if _POSIX:
+    from rpython.rlib.rmmap import (
+        MAP_PRIVATE, MAP_ANONYMOUS, PROT_EXEC, PROT_READ, PROT_WRITE, PTR)
+if _MS_WINDOWS:
+    from rpython.rlib.rwin32 import LPDWORD, DWORD, BOOL
+    from rpython.rlib.rmmap import (
+        MEM_COMMIT, MEM_RESERVE, PAGE_EXECUTE_READWRITE, MEM_RELEASE)
+
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib import rposix
+from rpython.rtyper.tool import rffi_platform
+
+
+locals().update(constants)
+
+
+def safe_external(name, args, result, save_err_on_unsafe=0, save_err_on_safe=0,
+                  **kwargs):
+    return rffi.llexternal(name, args, result,
+                           compilation_info=CConfig._compilation_info_,
+                           sandboxsafe=True, releasegil=False,
+                           save_err=save_err_on_safe, **kwargs)
+
+
+def safe_winexternal(name, args, result, **kwargs):
+    return rffi.llexternal(name, args, result,
+                           compilation_info=CConfig._compilation_info_,
+                           calling_conv='win', sandboxsafe=True,
+                           releasegil=False, **kwargs)
+
+
+if _POSIX:
+    c_mmap_safe = safe_external(
+        'mmap', [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], PTR,
+        macro=True, save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO)
+
+    c_munmap_safe = safe_external('munmap', [PTR, size_t], rffi.INT)
+
+
+if _CYGWIN:
+    c_free_safe = safe_external('free', [PTR], lltype.Void, macro=True)
+    c_malloc_safe = safe_external('malloc', [size_t], PTR, macro=True)
+
+if _MS_WINDOWS:
+    VirtualAlloc_safe = safe_winexternal(
+        'VirtualAlloc', [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD], rffi.VOIDP)
+
+    _VirtualProtect_safe = safe_winexternal(
+        'VirtualProtect', [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], BOOL)
+
+    def VirtualProtect(addr, size, mode, oldmode_ptr):
+        return _VirtualProtect_safe(
+            addr, rffi.cast(rffi.SIZE_T, size), rffi.cast(DWORD, mode),
+            oldmode_ptr)
+    VirtualProtect._annspecialcase_ = 'specialize:ll'
+
+    VirtualFree_safe = safe_winexternal(
+        'VirtualFree', [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL)
+
+
+if _POSIX:
+    def alloc_hinted(hintp, map_size):
+        flags = MAP_PRIVATE | MAP_ANONYMOUS
+        prot = PROT_EXEC | PROT_READ | PROT_WRITE
+        return c_mmap_safe(hintp, map_size, prot, flags, -1, 0)
+
+    # XXX is this really necessary?
+    class Hint:
+        pos = -0x4fff0000   # for reproducible results
+    hint = Hint()
+
+    def alloc(map_size):
+        """Allocate memory.  This is intended to be used by the JIT,
+        so the memory has the executable bit set and gets allocated
+        internally in case of a sandboxed process.
+        """
+        from errno import ENOMEM
+        from rpython.rlib import debug
+
+        if _CYGWIN:
+            # XXX: JIT memory should be using mmap MAP_PRIVATE with
+            #      PROT_EXEC but Cygwin's fork() fails.  mprotect()
+            #      cannot be used, but seems to be unnecessary there.
+            res = c_malloc_safe(map_size)
+            if res == rffi.cast(PTR, 0):
+                raise MemoryError
+            return res
+        res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size)
+        if res == rffi.cast(PTR, -1):
+            # some systems (some versions of OS/X?) complain if they
+            # are passed a non-zero address.  Try again.
+            res = alloc_hinted(rffi.cast(PTR, 0), map_size)
+            if res == rffi.cast(PTR, -1):
+                # ENOMEM simply raises MemoryError, but other errors are fatal
+                if rposix.get_saved_errno() != ENOMEM:
+                    debug.fatalerror_notb(
+                        "Got an unexpected error trying to allocate some "
+                        "memory for the JIT (tried to do mmap() with "
+                        "PROT_EXEC|PROT_READ|PROT_WRITE).  This can be caused "
+                        "by a system policy like PAX.  You need to find how "
+                        "to work around the policy on your system.")
+                raise MemoryError
+        else:
+            hint.pos += map_size
+        return res
+    alloc._annenforceargs_ = (int,)
+
+    if _CYGWIN:
+        free = c_free_safe
+    else:
+        free = c_munmap_safe
+
+elif _MS_WINDOWS:
+    class Hint:
+        pos = -0x4fff0000   # for reproducible results
+    hint = Hint()
+    # XXX this has no effect on windows
+
+    def alloc(map_size):
+        """Allocate memory.  This is intended to be used by the JIT,
+        so the memory has the executable bit set.
+        XXX implement me: it should get allocated internally in
+        case of a sandboxed process
+        """
+        null = lltype.nullptr(rffi.VOIDP.TO)
+        res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE,
+                                PAGE_EXECUTE_READWRITE)
+        if not res:
+            raise MemoryError
+        arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw')
+        VirtualProtect(res, map_size, PAGE_EXECUTE_READWRITE, arg)
+        lltype.free(arg, flavor='raw')
+        # ignore errors, just try
+        return res
+    alloc._annenforceargs_ = (int,)
+
+    def free(ptr, map_size):
+        VirtualFree_safe(ptr, 0, MEM_RELEASE)
diff --git a/rpython/jit/backend/x86/detect_feature.py b/rpython/jit/backend/x86/detect_feature.py
--- a/rpython/jit/backend/x86/detect_feature.py
+++ b/rpython/jit/backend/x86/detect_feature.py
@@ -1,17 +1,20 @@
 import sys
 import struct
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rmmap import alloc, free
+from rpython.rlib.rmmap import alloc, free, set_pages_executable
+
+CPU_INFO_SZ = 4096
 
 def cpu_info(instr):
-    data = alloc(4096)
+    data = alloc(CPU_INFO_SZ, no_exec=True)
     pos = 0
     for c in instr:
         data[pos] = c
         pos += 1
+    set_pages_executable(data, CPU_INFO_SZ)
     fnptr = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Signed)), data)
     code = fnptr()
-    free(data, 4096)
+    free(data, CPU_INFO_SZ)
     return code
 
 def detect_sse2():
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -155,6 +155,8 @@
     c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT,
                                    rffi.INT, off_t], PTR, macro=True,
                                    save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO)
+    c_mprotect, _ = external('mprotect',
+                             [PTR, size_t, rffi.INT], rffi.INT)
     # 'mmap' on linux32 is a macro that calls 'mmap64'
     _, c_munmap_safe = external('munmap', [PTR, size_t], rffi.INT)
     c_msync, _ = external('msync', [PTR, size_t, rffi.INT], rffi.INT,
@@ -705,14 +707,21 @@
         m.setdata(res, map_size)
         return m
 
-    def alloc_hinted(hintp, map_size):
+    def alloc_hinted(hintp, map_size, no_exec=False):
         flags = MAP_PRIVATE | MAP_ANONYMOUS
-        prot = PROT_EXEC | PROT_READ | PROT_WRITE
+        prot = PROT_READ | PROT_WRITE
+        if not no_exec:
+            prot |= PROT_EXEC
         if we_are_translated():
             flags = NonConstant(flags)
             prot = NonConstant(prot)
         return c_mmap_safe(hintp, map_size, prot, flags, -1, 0)
 
+    def set_pages_executable(addr, size):
+        rv = c_mprotect(addr, size, PROT_EXEC)
+        if rv < 0:
+            debug.fatalerror_notb("set_pages_executable failed")
+
     def clear_large_memory_chunk_aligned(addr, map_size):
         addr = rffi.cast(PTR, addr)
         flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS
@@ -728,7 +737,7 @@
         pos = -0x4fff0000   # for reproducible results
     hint = Hint()
 
-    def alloc(map_size):
+    def alloc(map_size, no_exec=False):
         """Allocate memory.  This is intended to be used by the JIT,
         so the memory has the executable bit set and gets allocated
         internally in case of a sandboxed process.
@@ -744,11 +753,11 @@
             if res == rffi.cast(PTR, 0):
                 raise MemoryError
             return res
-        res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size)
+        res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size, no_exec=no_exec)
         if res == rffi.cast(PTR, -1):
             # some systems (some versions of OS/X?) complain if they
             # are passed a non-zero address.  Try again.
-            res = alloc_hinted(rffi.cast(PTR, 0), map_size)
+            res = alloc_hinted(rffi.cast(PTR, 0), map_size, no_exec=no_exec)
             if res == rffi.cast(PTR, -1):
                 # ENOMEM simply raises MemoryError, but other errors are fatal
                 if rposix.get_saved_errno() != ENOMEM:
@@ -762,7 +771,7 @@
         else:
             hint.pos += map_size
         return res
-    alloc._annenforceargs_ = (int,)
+    alloc._annenforceargs_ = (int, bool)
 
     if _CYGWIN:
         free = c_free_safe
@@ -933,11 +942,13 @@
     hint = Hint()
     # XXX this has no effect on windows
 
-    def alloc(map_size):
+    def alloc(map_size, no_exec=no_exec):
         """Allocate memory.  This is intended to be used by the JIT,
         so the memory has the executable bit set.
         XXX implement me: it should get allocated internally in
         case of a sandboxed process
+
+        XXX no_exec not implemented on windows
         """
         null = lltype.nullptr(rffi.VOIDP.TO)
         res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE,
@@ -949,7 +960,10 @@
         lltype.free(arg, flavor='raw')
         # ignore errors, just try
         return res
-    alloc._annenforceargs_ = (int,)
+    alloc._annenforceargs_ = (int, bool)
+
+    def set_pages_executable(addr, size):
+        pass # XXX not implemented on windows
 
     def free(ptr, map_size):
         VirtualFree_safe(ptr, 0, MEM_RELEASE)

From pypy.commits at gmail.com  Fri Aug 12 09:01:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 06:01:30 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Emulate modf. Fix emulation of
 dtoa(2.0) to output the ".0" too.
Message-ID: <57adc8aa.10a81c0a.b1ccd.e1a1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86170:44f0653642eb
Date: 2016-08-12 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/44f0653642eb/

Log:	Emulate modf. Fix emulation of dtoa(2.0) to output the ".0" too.

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -232,3 +232,7 @@
     s = hlstr(s)
     assert s is not None
     return s
+
+def emulate_modf(x):
+    return (llop.revdb_modf(lltype.Float, x, 0),
+            llop.revdb_modf(lltype.Float, x, 1))
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -586,6 +586,7 @@
     'revdb_set_thread_breakpoint': LLOp(),
     'revdb_strtod':         LLOp(sideeffects=False),
     'revdb_dtoa':           LLOp(sideeffects=False),
+    'revdb_modf':           LLOp(sideeffects=False),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py
--- a/rpython/rtyper/lltypesystem/module/ll_math.py
+++ b/rpython/rtyper/lltypesystem/module/ll_math.py
@@ -4,7 +4,7 @@
 import sys
 
 from rpython.translator import cdir
-from rpython.rlib import jit, rposix
+from rpython.rlib import jit, rposix, revdb
 from rpython.rlib.rfloat import INFINITY, NAN, isfinite, isinf, isnan
 from rpython.rlib.rposix import UNDERSCORE_ON_WIN32
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -221,6 +221,8 @@
 def ll_math_modf(x):
     # some platforms don't do the right thing for NaNs and
     # infinities, so we take care of special cases directly.
+    if revdb.flag_io_disabled():
+        return revdb.emulate_modf(x)
     if not isfinite(x):
         if isnan(x):
             return (x, x)
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1759,11 +1759,18 @@
 
 RPY_EXTERN RPyString *rpy_reverse_db_dtoa(double d)
 {
-    char buffer[128];
+    char buffer[128], *p;
     RPyString *result;
     int size;
-    size = snprintf(buffer, sizeof(buffer), "%g", d);
-    if (size < 0) size = 0;   /* XXX? */
+    size = snprintf(buffer, sizeof(buffer) - 3, "%g", d);
+    if (size < 0)
+        size = 0;
+    for (p = buffer; '0' <= *p && *p <= '9'; p++) {
+    }
+    if (*p == 0) {    /* a pure integer */
+        buffer[size++] = '.';
+        buffer[size++] = '0';
+    }
     result = make_rpy_string(size);
     memcpy(_RPyString_AsString(result), buffer, size);
     return result;
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -236,6 +236,13 @@
 #define OP_REVDB_DTOA(d, r)                                             \
     r = rpy_reverse_db_dtoa(d)
 
+#define OP_REVDB_MODF(x, index, r)                                      \
+    do {                                                                \
+        double _r0, _r1;                                                \
+        _r0 = modf(x, &_r1);                                            \
+        r = (index == 0) ? _r0 : _r1;                                   \
+    } while (0)
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -1,4 +1,4 @@
-import py, sys
+import py, sys, math
 from cStringIO import StringIO
 from rpython.rlib import revdb, rdtoa
 from rpython.rlib.debug import debug_print, ll_assert
@@ -49,7 +49,9 @@
                 stuff = dbstate.metavar
             elif extra == '2.35':
                 val = rdtoa.strtod('2.35')
-                revdb.send_output(rdtoa.dtoa(val))
+                valx, valy = math.modf(val)
+                revdb.send_output(rdtoa.dtoa(valx) + '\n')
+                revdb.send_output(rdtoa.dtoa(valy) + '\n')
                 return
             else:
                 assert False
@@ -208,4 +210,4 @@
         group = ReplayProcessGroup(str(self.exename), self.rdbname)
         with stdout_capture() as buf:
             group.print_cmd('2.35')
-        assert buf.getvalue() == "2.35"
+        assert buf.getvalue() == "0.35\n2.0\n"

From pypy.commits at gmail.com  Fri Aug 12 10:13:10 2016
From: pypy.commits at gmail.com (vext01)
Date: Fri, 12 Aug 2016 07:13:10 -0700 (PDT)
Subject: [pypy-commit] pypy refactor_rmmap: Close branch.
Message-ID: <57add976.6aaac20a.437bd.c571@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: refactor_rmmap
Changeset: r86171:9f7d18f5d82f
Date: 2016-08-12 15:12 +0100
http://bitbucket.org/pypy/pypy/changeset/9f7d18f5d82f/

Log:	Close branch.


From pypy.commits at gmail.com  Fri Aug 12 11:49:30 2016
From: pypy.commits at gmail.com (mjacob)
Date: Fri, 12 Aug 2016 08:49:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57adf00a.a717c20a.4589.e4c0@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r86172:d1ba25403058
Date: 2016-08-12 17:48 +0200
http://bitbucket.org/pypy/pypy/changeset/d1ba25403058/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 4110 lines

diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -105,3 +105,26 @@
 .. branch: ep2016sprint
 
 Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -114,8 +114,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1322,6 +1322,25 @@
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
 
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts
+
     def test_call_function_var(self):
         source = """call(*me)"""
         code, blocks = generate_function_code(source, self.space)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1801,6 +1801,40 @@
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
         """)
 
+    def resource_warning(self, w_msg, w_tb):
+        self.appexec([w_msg, w_tb],
+                     """(msg, tb):
+            import sys
+            print >> sys.stderr, msg
+            if tb:
+                print >> sys.stderr, "Created at (most recent call last):"
+                print >> sys.stderr, tb
+        """)
+
+    def format_traceback(self):
+        # we need to disable track_resources before calling the traceback
+        # module. Else, it tries to open more files to format the traceback,
+        # the file constructor will call space.format_traceback etc., in an
+        # inifite recursion
+        flag = self.sys.track_resources
+        self.sys.track_resources = False
+        try:
+            return self.appexec([],
+                         """():
+                import sys, traceback
+                # the "1" is because we don't want to show THIS code
+                # object in the traceback
+                try:
+                    f = sys._getframe(1)
+                except ValueError:
+                    # this happens if you call format_traceback at the very beginning
+                    # of startup, when there is no bottom code object
+                    return '<no stacktrace available>'
+                return "".join(traceback.format_stack(f))
+            """)
+        finally:
+            self.sys.track_resources = flag
+
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -209,6 +209,13 @@
         self.check(['-c', 'pass'], {'PYTHONNOUSERSITE': '1'}, sys_argv=['-c'],
                    run_command='pass', **expected)
 
+    def test_track_resources(self, monkeypatch):
+        myflag = [False]
+        def pypy_set_track_resources(flag):
+            myflag[0] = flag
+        monkeypatch.setattr(sys, 'pypy_set_track_resources', pypy_set_track_resources, raising=False)
+        self.check(['-X', 'track-resources'], {}, sys_argv=[''], run_stdin=True)
+        assert myflag[0] == True
 
 class TestInteraction:
     """
@@ -1152,4 +1159,3 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -458,3 +458,28 @@
         space.finish()
         # assert that we reach this point without getting interrupted
         # by the OperationError(NameError)
+
+    def test_format_traceback(self):
+        from pypy.tool.pytest.objspace import maketestobjspace
+        from pypy.interpreter.gateway import interp2app
+        #
+        def format_traceback(space):
+            return space.format_traceback()
+        #
+        space = maketestobjspace()
+        w_format_traceback = space.wrap(interp2app(format_traceback))
+        w_tb = space.appexec([w_format_traceback], """(format_traceback):
+            def foo():
+                return bar()
+            def bar():
+                return format_traceback()
+            return foo()
+        """)
+        tb = space.str_w(w_tb)
+        expected = '\n'.join([
+            '  File "?", line 6, in anonymous',  # this is the appexec code object
+            '  File "?", line 3, in foo',
+            '  File "?", line 5, in bar',
+            ''
+        ])
+        assert tb == expected
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.7.0"
+VERSION = "1.8.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -157,11 +157,13 @@
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
         try:
+            keepalives = [None] * len(args_w)    # None or strings
             for i in range(len(args_w)):
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                 w_obj = args_w[i]
                 argtype = self.fargs[i]
-                if argtype.convert_argument_from_object(data, w_obj):
+                if argtype.convert_argument_from_object(data, w_obj,
+                                                        keepalives, i):
                     # argtype is a pointer type, and w_obj a list/tuple/str
                     mustfree_max_plus_1 = i + 1
 
@@ -177,9 +179,13 @@
                 if isinstance(argtype, W_CTypePointer):
                     data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                     flag = get_mustfree_flag(data)
+                    raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                     if flag == 1:
-                        raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
+                    elif flag >= 4:
+                        value = keepalives[i]
+                        assert value is not None
+                        rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
             lltype.free(buffer, flavor='raw')
             keepalive_until_here(args_w)
         return w_res
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -83,7 +83,7 @@
         raise oefmt(space.w_TypeError, "cannot initialize cdata '%s'",
                     self.name)
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         self.convert_from_object(cdata, w_ob)
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -16,8 +16,8 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
+    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
     length = -1
 
     def __init__(self, space, size, extra, extra_position, ctitem,
@@ -30,6 +30,9 @@
         #  - for functions, it is the return type
         self.ctitem = ctitem
         self.can_cast_anything = could_cast_anything and ctitem.cast_anything
+        self.accept_str = (self.can_cast_anything or
+                            (ctitem.is_primitive_integer and
+                             ctitem.size == rffi.sizeof(lltype.Char)))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -72,9 +75,7 @@
                 pass
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
-        elif (self.can_cast_anything or
-              (self.ctitem.is_primitive_integer and
-               self.ctitem.size == rffi.sizeof(lltype.Char))):
+        elif self.accept_str:
             if not space.isinstance_w(w_ob, space.w_str):
                 raise self._convert_error("bytes or list or tuple", w_ob)
             s = space.str_w(w_ob)
@@ -262,8 +263,16 @@
         else:
             return lltype.nullptr(rffi.CCHARP.TO)
 
-    def _prepare_pointer_call_argument(self, w_init, cdata):
+    def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
         space = self.space
+        if self.accept_str and space.isinstance_w(w_init, space.w_str):
+            # special case to optimize strings passed to a "char *" argument
+            value = w_init.str_w(space)
+            keepalives[i] = value
+            buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
+            rffi.cast(rffi.CCHARPP, cdata)[0] = buf
+            return ord(buf_flag)    # 4, 5 or 6
+        #
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
@@ -300,10 +309,11 @@
         rffi.cast(rffi.CCHARPP, cdata)[0] = result
         return 1
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
         result = (not isinstance(w_ob, cdataobj.W_CData) and
-                  self._prepare_pointer_call_argument(w_ob, cdata))
+                  self._prepare_pointer_call_argument(w_ob, cdata,
+                                                      keepalives, i))
         if result == 0:
             self.convert_from_object(cdata, w_ob)
         set_mustfree_flag(cdata, result)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -353,7 +353,7 @@
 'array.array' or numpy arrays."""
         #
         w_ctchara = newtype._new_chara_type(self.space)
-        return func.from_buffer(self.space, w_ctchara, w_python_buffer)
+        return func._from_buffer(self.space, w_ctchara, w_python_buffer)
 
 
     @unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated
+from rpython.rlib import jit
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -132,17 +133,66 @@
         raise oefmt(space.w_TypeError,
                     "needs 'char[]', got '%s'", w_ctype.name)
     #
+    return _from_buffer(space, w_ctype, w_x)
+
+def _from_buffer(space, w_ctype, w_x):
     buf = _fetch_as_read_buffer(space, w_x)
-    try:
-        _cdata = buf.get_raw_address()
-    except ValueError:
-        raise oefmt(space.w_TypeError,
-                    "from_buffer() got a '%T' object, which supports the "
-                    "buffer interface but cannot be rendered as a plain "
-                    "raw address on PyPy", w_x)
+    if space.isinstance_w(w_x, space.w_str):
+        _cdata = get_raw_address_of_string(space, w_x)
+    else:
+        try:
+            _cdata = buf.get_raw_address()
+        except ValueError:
+            raise oefmt(space.w_TypeError,
+                        "from_buffer() got a '%T' object, which supports the "
+                        "buffer interface but cannot be rendered as a plain "
+                        "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
 
+# ____________________________________________________________
+
+class RawBytes(object):
+    def __init__(self, string):
+        self.ptr = rffi.str2charp(string, track_allocation=False)
+    def __del__(self):
+        rffi.free_charp(self.ptr, track_allocation=False)
+
+class RawBytesCache(object):
+    def __init__(self, space):
+        from pypy.interpreter.baseobjspace import W_Root
+        from rpython.rlib import rweakref
+        self.wdict = rweakref.RWeakKeyDictionary(W_Root, RawBytes)
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(space, w_x):
+    """Special case for ffi.from_buffer(string).  Returns a 'char *' that
+    is valid as long as the string object is alive.  Two calls to
+    ffi.from_buffer(same_string) are guaranteed to return the same pointer.
+    """
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rtyper.lltypesystem.rstr import STR
+    from rpython.rtyper.lltypesystem import llmemory
+    from rpython.rlib import rgc
+
+    cache = space.fromcache(RawBytesCache)
+    rawbytes = cache.wdict.get(w_x)
+    if rawbytes is None:
+        data = space.str_w(w_x)
+        if we_are_translated() and not rgc.can_move(data):
+            lldata = llstr(data)
+            data_start = (llmemory.cast_ptr_to_adr(lldata) +
+                          rffi.offsetof(STR, 'chars') +
+                          llmemory.itemoffsetof(STR.chars, 0))
+            data_start = rffi.cast(rffi.CCHARP, data_start)
+            data_start[len(data)] = '\x00'   # write the final extra null
+            return data_start
+        rawbytes = RawBytes(data)
+        cache.wdict.set(w_x, rawbytes)
+    return rawbytes.ptr
+
+# ____________________________________________________________
+
 
 def unsafe_escaping_ptr_for_ptr_or_array(w_cdata):
     if not w_cdata.ctype.is_nonfunc_pointer_or_array:
diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py
--- a/pypy/module/_cffi_backend/parse_c_type.py
+++ b/pypy/module/_cffi_backend/parse_c_type.py
@@ -97,11 +97,8 @@
                                   [rffi.INT], rffi.CCHARP)
 
 def parse_c_type(info, input):
-    p_input = rffi.str2charp(input)
-    try:
+    with rffi.scoped_view_charp(input) as p_input:
         res = ll_parse_c_type(info, p_input)
-    finally:
-        rffi.free_charp(p_input)
     return rffi.cast(lltype.Signed, res)
 
 NULL_CTX = lltype.nullptr(PCTX.TO)
@@ -130,15 +127,13 @@
     return rffi.getintfield(src_ctx, 'c_num_types')
 
 def search_in_globals(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_globals(ctx, c_name,
-                                  rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_globals(ctx, c_name,
+                                      rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
 
 def search_in_struct_unions(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_struct_unions(ctx, c_name,
-                                        rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_struct_unions(ctx, c_name,
+                                            rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3330,13 +3330,18 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharA = new_array_type(BCharP, None)
-    py.test.raises(TypeError, from_buffer, BCharA, b"foo")
+    p1 = from_buffer(BCharA, b"foo")
+    assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # from_buffer(buffer(b"foo")) does not work, because it's not
+        # implemented on pypy; only from_buffer(b"foo") works.
         py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
         py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
     try:
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -402,21 +402,20 @@
             _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
         from rpython.rlib import rwin32
 
-        charp = rffi.str2charp(buf)
-        written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                    flavor='raw')
-        try:
-            result = _WriteFile(
-                self.handle, rffi.ptradd(charp, offset),
-                size, written_ptr, rffi.NULL)
+        with rffi.scoped_view_charp(buf) as charp:
+            written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
+                                        flavor='raw')
+            try:
+                result = _WriteFile(
+                    self.handle, rffi.ptradd(charp, offset),
+                    size, written_ptr, rffi.NULL)
 
-            if (result == 0 and
-                rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
-                raise oefmt(space.w_ValueError,
-                            "Cannot send %d bytes over connection", size)
-        finally:
-            rffi.free_charp(charp)
-            lltype.free(written_ptr, flavor='raw')
+                if (result == 0 and
+                    rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
+                    raise oefmt(space.w_ValueError,
+                                "Cannot send %d bytes over connection", size)
+            finally:
+                lltype.free(written_ptr, flavor='raw')
 
     def do_recv_string(self, space, buflength, maxlength):
         from pypy.module._multiprocessing.interp_win32 import (
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -15,8 +15,9 @@
 
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
-        includes=['unistd.h', 'sys/syscall.h'])
+        includes=['unistd.h', 'sys/syscall.h', 'sys/stat.h'])
     HAVE_SYS_SYSCALL_H = platform.Has("syscall")
+    HAVE_SYS_STAT_H = platform.Has("stat")
     HAVE_SETSID = platform.Has("setsid")
 
 config = platform.configure(CConfig)
@@ -29,6 +30,8 @@
 compile_extra = []
 if config['HAVE_SYS_SYSCALL_H']:
     compile_extra.append("-DHAVE_SYS_SYSCALL_H")
+if config['HAVE_SYS_STAT_H']:
+    compile_extra.append("-DHAVE_SYS_STAT_H")
 if config['HAVE_SETSID']:
     compile_extra.append("-DHAVE_SETSID")
 
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,6 +1,7 @@
 import sys, os
-import py
+import pytest
 from pypy.tool.pytest.objspace import gettestobjspace
+from pypy.interpreter.gateway import interp2app
 from rpython.tool.udir import udir
 from rpython.rlib import rsocket
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -13,8 +14,6 @@
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
     mod.path = udir.join('fd')
     mod.path.write('fo')
-    mod.raises = py.test.raises # make raises available from app-level tests
-    mod.skip = py.test.skip
 
 def test_gethostname():
     host = space.appexec([w_socket], "(_socket): return _socket.gethostname()")
@@ -42,7 +41,7 @@
     for host in ["localhost", "127.0.0.1", "::1"]:
         if host == "::1" and not ipv6:
             from pypy.interpreter.error import OperationError
-            with py.test.raises(OperationError):
+            with pytest.raises(OperationError):
                 space.appexec([w_socket, space.wrap(host)],
                               "(_socket, host): return _socket.gethostbyaddr(host)")
             continue
@@ -58,14 +57,14 @@
     assert space.unwrap(port) == 25
     # 1 arg version
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyname second argument is not optional before python 2.4")
+        pytest.skip("getservbyname second argument is not optional before python 2.4")
     port = space.appexec([w_socket, space.wrap(name)],
                         "(_socket, name): return _socket.getservbyname(name)")
     assert space.unwrap(port) == 25
 
 def test_getservbyport():
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyport does not exist before python 2.4")
+        pytest.skip("getservbyport does not exist before python 2.4")
     port = 25
     # 2 args version
     name = space.appexec([w_socket, space.wrap(port)],
@@ -139,7 +138,7 @@
 
 def test_pton_ntop_ipv4():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     tests = [
         ("123.45.67.89", "\x7b\x2d\x43\x59"),
         ("0.0.0.0", "\x00" * 4),
@@ -155,9 +154,9 @@
 
 def test_ntop_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -176,9 +175,9 @@
 
 def test_pton_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -197,7 +196,7 @@
         assert space.unwrap(w_packed) == packed
 
 def test_has_ipv6():
-    py.test.skip("has_ipv6 is always True on PyPy for now")
+    pytest.skip("has_ipv6 is always True on PyPy for now")
     res = space.appexec([w_socket], "(_socket): return _socket.has_ipv6")
     assert space.unwrap(res) == socket.has_ipv6
 
@@ -231,7 +230,7 @@
 def test_addr_raw_packet():
     from pypy.module._socket.interp_socket import addr_as_object
     if not hasattr(rsocket._c, 'sockaddr_ll'):
-        py.test.skip("posix specific test")
+        pytest.skip("posix specific test")
     # HACK: To get the correct interface number of lo, which in most cases is 1,
     # but can be anything (i.e. 39), we need to call the libc function
     # if_nametoindex to get the correct index
@@ -653,11 +652,11 @@
 class AppTestNetlink:
     def setup_class(cls):
         if not hasattr(os, 'getpid'):
-            py.test.skip("AF_NETLINK needs os.getpid()")
+            pytest.skip("AF_NETLINK needs os.getpid()")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_NETLINK')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_NETLINK on this platform")
+            pytest.skip("no AF_NETLINK on this platform")
         cls.space = space
 
     def test_connect_to_kernel_netlink_routing_socket(self):
@@ -673,11 +672,11 @@
 class AppTestPacket:
     def setup_class(cls):
         if not hasattr(os, 'getuid') or os.getuid() != 0:
-            py.test.skip("AF_PACKET needs to be root for testing")
+            pytest.skip("AF_PACKET needs to be root for testing")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_PACKET')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_PACKET on this platform")
+            pytest.skip("no AF_PACKET on this platform")
         cls.space = space
 
     def test_convert_between_tuple_and_sockaddr_ll(self):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -146,7 +146,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
@@ -158,7 +158,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
@@ -192,7 +192,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
@@ -204,7 +204,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
@@ -239,7 +239,7 @@
 
         Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
         bound on the entropy contained in string."""
-        with rffi.scoped_str2charp(string) as buf:
+        with rffi.scoped_nonmovingbuffer(string) as buf:
             libssl_RAND_add(buf, len(string), entropy)
 
     def _RAND_bytes(space, n, pseudo):
diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -537,9 +537,8 @@
     releasegil=ts_helper,
     compilation_info=backend.eci)
 def c_charp2stdstring(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2stdstring(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2stdstring(charp)
     return result
 _c_stdstring2stdstring = rffi.llexternal(
     "cppyy_stdstring2stdstring",
diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -82,9 +82,8 @@
     releasegil=ts_helper,
     compilation_info=eci)
 def c_charp2TString(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2TString(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2TString(charp)
     return result
 _c_TString2TString = rffi.llexternal(
     "cppyy_TString2TString",
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -65,6 +65,7 @@
                 else:    # only other use is sring
                     n = len(obj._string)
                     assert raw_string == rffi.cast(rffi.CCHARP, 0)
+                    # XXX could use rffi.get_nonmovingbuffer_final_null()
                     raw_string = rffi.str2charp(obj._string)
                     data = rffi.cast(rffi.CCHARPP, data)
                     data[0] = raw_string
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,6 +11,9 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+        c_fdopen, c_fileno,
+        c_fopen)# for tests
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import NameManager
@@ -84,44 +87,32 @@
 assert CONST_WSTRING == rffi.CWCHARP
 
 # FILE* interface
-FILEP = rffi.COpaquePtr('FILE')
 
 if sys.platform == 'win32':
     dash = '_'
 else:
     dash = ''
-fileno = rffi.llexternal(dash + 'fileno', [FILEP], rffi.INT)
-fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
-fdopen = rffi.llexternal(dash + 'fdopen', [rffi.INT, CONST_STRING],
-                  FILEP, save_err=rffi.RFFI_SAVE_ERRNO)
 
-_fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
-    if not is_valid_fd(fileno(fp)):
+    if not is_valid_fd(c_fileno(fp)):
         return -1
-    return _fclose(fp)
+    return c_fclose(fp)
 
-_fwrite = rffi.llexternal('fwrite',
-                         [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                         rffi.SIZE_T)
 def fwrite(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fwrite(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fwrite(buf, sz, n, fp)
 
-_fread = rffi.llexternal('fread',
-                        [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                        rffi.SIZE_T)
 def fread(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fread(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fread(buf, sz, n, fp)
 
 _feof = rffi.llexternal('feof', [FILEP], rffi.INT)
 def feof(fp):
-    validate_fd(fileno(fp))
+    validate_fd(c_fileno(fp))
     return _feof(fp)
 
 def is_valid_fp(fp):
-    return is_valid_fd(fileno(fp))
+    return is_valid_fd(c_fileno(fp))
 
 pypy_decl = 'pypy_decl.h'
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -96,7 +96,8 @@
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
             py_str.c_ob_size, len(s))
-    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    with rffi.scoped_nonmovingbuffer(s) as s_ptr:
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
     py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
diff --git a/pypy/module/cpyext/c-api.txt b/pypy/module/cpyext/c-api.txt
deleted file mode 100644
--- a/pypy/module/cpyext/c-api.txt
+++ /dev/null
@@ -1,71 +0,0 @@
-Reference Count
-===============
-
-XXX
-
-Borrowed References
-===================
-
-XXX
-
-PyStringObject support
-======================
-
-The problem
------------
-
-PyString_AsString() returns a (non-movable) pointer to the underlying
-buffer, whereas pypy strings are movable.  C code may temporarily
-store this address and use it, as long as it owns a reference to the
-PyObject.  There is no "release" function to specify that the pointer
-is not needed any more.
-
-Note that the pointer may be used to fill the initial value of
-string. This is valid only when the string was just allocated, and is
-not used elsewhere.
-
-Proposed solution
------------------
-
-Our emulation of the PyStringObject contains an additional member: a
-pointer to a char buffer; it may be NULL.
-
-- A string allocated by pypy will be converted into a PyStringObject
-  with a NULL buffer.  When PyString_AsString() is called, memory is
-  allocated (with flavor='raw') and content is copied.
-
-- A string allocated with PyString_FromStringAndSize(NULL, size) will
-  allocate a buffer with the specified size, but the reference won't
-  be stored in the global map py_objects_r2w; there won't be a
-  corresponding object in pypy.  When from_ref() or Py_INCREF() is
-  called, the pypy string is created, and added in py_objects_r2w.
-  The buffer is then supposed to be immutable.
-
-- _PyString_Resize works only on not-yet-pypy'd strings, and returns a
-  similar object.
-
-- PyString_Size don't need to force the object. (in this case, another
-  "size" member is needed)
-
-- There could be an (expensive!) check in from_ref() that the buffer
-  still corresponds to the pypy gc-managed string.
-
-PySequence_Fast support
-======================
-There are five functions for fast sequence access offered by the CPython API:
-
-PyObject* PySequence_Fast(PyObject *o, const char *m)
-
-PyObject* PySequence_Fast_GET_ITEM(	PyObject *o, int i)
-
-PyObject** PySequence_Fast_ITEMS(	PyObject *o)
-
-PyObject* PySequence_ITEM(	PyObject *o, int i)
-
-int PySequence_Fast_GET_SIZE(	PyObject *o)
-
-PyPy supports four of these, but does not support PySequence_Fast_ITEMS.
-(Various ways to support PySequence_Fast_ITEMS were considered. They all had
-two things in common: they would have taken a lot of work, and they would have
-resulted in incomplete semantics or in poor performance. We decided that a slow
-implementation of PySequence_Fast_ITEMS was not very useful.)
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -25,6 +25,8 @@
                          flavor='raw',
                          add_memory_pressure=True)
 
+realloc = rffi.llexternal('realloc', [rffi.VOIDP, rffi.SIZE_T], rffi.VOIDP)
+
 @cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
@@ -32,7 +34,7 @@
                          flavor='raw',
                          add_memory_pressure=True)
     # XXX FIXME
-    return lltype.nullptr(rffi.VOIDP.TO)
+    return realloc(ptr, size)
 
 @cpython_api([rffi.VOIDP], lltype.Void)
 def PyObject_Free(space, ptr):
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -10,7 +10,7 @@
 from pypy.objspace.std import tupleobject
 
 from pypy.module.cpyext.tupleobject import PyTuple_Check, PyTuple_SetItem
-from pypy.module.cpyext.object import Py_IncRef, Py_DecRef
+from pypy.module.cpyext.pyobject import decref
 
 from pypy.module.cpyext.dictobject import PyDict_Check
 
@@ -252,7 +252,7 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        Py_DecRef(w_list.space, storage._elems[index])
+        decref(w_list.space, storage._elems[index])
         storage._elems[index] = make_ref(w_list.space, w_obj)
 
     def length(self, w_list):
@@ -264,9 +264,8 @@
         return storage._elems
 
     def getslice(self, w_list, start, stop, step, length):
-        #storage = self.unerase(w_list.lstorage)
-        raise oefmt(w_list.space.w_NotImplementedError,
-                    "settting a slice of a PySequence_Fast is not supported")
+        w_list.switch_to_object_strategy()
+        return w_list.strategy.getslice(w_list, start, stop, step, length)
 
     def getitems(self, w_list):
         # called when switching list strategy, so convert storage
@@ -389,5 +388,5 @@
 
     def __del__(self):
         for i in range(self._length):
-            Py_DecRef(self.space, self._elems[i])
+            decref(self.space, self._elems[i])
         lltype.free(self._elems, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.eval import (
     Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP
+from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
@@ -130,19 +130,19 @@
     def test_run_file(self, space, api):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
-        fp = fopen(str(filepath), "rb")
+        fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_ZeroDivisionError
         api.PyErr_Clear()
 
         # try again, but with a closed file
-        fp = fopen(str(filepath), "rb")
-        os.close(fileno(fp))
+        fp = c_fopen(str(filepath), "rb")
+        os.close(c_fileno(fp))
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -212,8 +212,9 @@
         assert type(x) is float
         assert x == -12.34
 
-    @pytest.mark.skipif(True, reason='realloc not fully implemented')
     def test_object_realloc(self):
+        if not self.runappdirect:
+            skip('no untranslated support for realloc')
         module = self.import_extension('foo', [
             ("realloctest", "METH_NOARGS",
              """
@@ -221,12 +222,11 @@
                  char *copy, *orig = PyObject_MALLOC(12);
                  memcpy(orig, "hello world", 12);
                  copy = PyObject_REALLOC(orig, 15);
+                 /* realloc() takes care of freeing orig, if changed */
                  if (copy == NULL)
                      Py_RETURN_NONE;
                  ret = PyBytes_FromStringAndSize(copy, 12);
-                 if (copy != orig)
-                     PyObject_Free(copy);
-                 PyObject_Free(orig);
+                 PyObject_Free(copy);
                  return ret;
              """)])
         x = module.realloctest()
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -78,6 +78,17 @@
         assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
         assert space.eq_w(w_t, space.wrap([1, 3, 5]))
 
+    def test_get_slice_fast(self, space, api):
+        w_t = space.wrap([1, 2, 3, 4, 5])
+        api.PySequence_Fast(w_t, "foo") # converts
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4]
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4]
+
+        assert api.PySequence_DelSlice(w_t, 1, 4) == 0
+        assert space.eq_w(w_t, space.wrap([1, 5]))
+        assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
+        assert space.eq_w(w_t, space.wrap([1, 3, 5]))
+
     def test_iter(self, space, api):
         w_t = space.wrap((1, 2))
         w_iter = api.PySeqIter_New(w_t)
@@ -226,18 +237,33 @@
         assert space.int_w(space.len(w_l)) == 10
 
 
-class XAppTestSequenceObject(AppTestCpythonExtensionBase):
-    def test_sequenceobject(self):
+class AppTestSequenceObject(AppTestCpythonExtensionBase):
+    def test_fast(self):
         module = self.import_extension('foo', [
             ("test_fast_sequence", "METH_VARARGS",
              """
-                PyObject * o = PyTuple_GetItem(args, 0);
+                int size, i;
+                PyTypeObject * common_type;
+                PyObject *foo, **objects;
+                PyObject * seq = PyTuple_GetItem(args, 0);
                 /* XXX assert it is a tuple */
-                PyObject *foo = PySequence_Fast(o, "some string");
-                PyObject ** res = PySequence_Fast_ITEMS(foo);
-                /* XXX do some kind of test on res */
-                /* XXX now what? who manages res's refcount? */
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "some string");
+                objects = PySequence_Fast_ITEMS(foo);
+                size = PySequence_Fast_GET_SIZE(seq);
+                common_type = size > 0 ? Py_TYPE(objects[0]) : NULL;
+                for (i = 1; i < size; ++i) {
+                    if (Py_TYPE(objects[i]) != common_type) {
+                        common_type = NULL;
+                        break;
+                    }
+                }
+                Py_DECREF(foo);
+                Py_DECREF(common_type);
                 return PyBool_FromLong(1);
              """)])
-        assert module.test_fast_sequence([1, 2, 3, 4])
+        s = [1, 2, 3, 4]
+        assert module.test_fast_sequence(s[0:-1])
+        assert module.test_fast_sequence(s[::-1])
 
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -47,6 +47,7 @@
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
+    from rpython.tool.error import offset2lineno
     bcindex = ord(bytecode.co_code[next_instr])
     opname = ""
     if 0 <= bcindex < len(opcode_method_names):
@@ -54,7 +55,8 @@
     name = bytecode.co_name
     if not name:
         name = ""
-    return (bytecode.co_filename, bytecode.co_firstlineno,
+    line = offset2lineno(bytecode, intmask(next_instr))
+    return (bytecode.co_filename, line,
             name, intmask(next_instr), opname)
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -19,6 +19,7 @@
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
         self.debug = True
+        self.track_resources = False
         self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
@@ -48,6 +49,8 @@
         '_current_frames'       : 'currentframes._current_frames',
         'setrecursionlimit'     : 'vm.setrecursionlimit',
         'getrecursionlimit'     : 'vm.getrecursionlimit',
+        'pypy_set_track_resources' : 'vm.set_track_resources',
+        'pypy_get_track_resources' : 'vm.get_track_resources',
         'setcheckinterval'      : 'vm.setcheckinterval',
         'getcheckinterval'      : 'vm.getcheckinterval',
         'exc_info'              : 'vm.exc_info',
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -61,6 +61,13 @@
     """
     return space.wrap(space.sys.recursionlimit)
 
+ at unwrap_spec(flag=bool)
+def set_track_resources(space, flag):
+    space.sys.track_resources = flag
+
+def get_track_resources(space):
+    return space.wrap(space.sys.track_resources)
+
 @unwrap_spec(interval=int)
 def setcheckinterval(space, interval):
     """Tell the Python interpreter to check for asynchronous events every
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -130,7 +130,7 @@
                 cls.module = str(udir.join('testownlib.dll'))
         else:
             subprocess.check_call(
-                'gcc testownlib.c -shared -fPIC -o testownlib.so',
+                'cc testownlib.c -shared -fPIC -o testownlib.so',
                 cwd=str(udir), shell=True)
             cls.module = str(udir.join('testownlib.so'))
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -852,9 +852,12 @@
     assert str(e2.value) == "foo0() takes no arguments (2 given)"
     assert str(e3.value) == "foo1() takes exactly one argument (0 given)"
     assert str(e4.value) == "foo1() takes exactly one argument (2 given)"
-    assert str(e5.value) == "foo2() takes exactly 2 arguments (0 given)"
-    assert str(e6.value) == "foo2() takes exactly 2 arguments (1 given)"
-    assert str(e7.value) == "foo2() takes exactly 2 arguments (3 given)"
+    assert str(e5.value) in ["foo2 expected 2 arguments, got 0",
+                             "foo2() takes exactly 2 arguments (0 given)"]
+    assert str(e6.value) in ["foo2 expected 2 arguments, got 1",
+                             "foo2() takes exactly 2 arguments (1 given)"]
+    assert str(e7.value) in ["foo2 expected 2 arguments, got 3",
+                             "foo2() takes exactly 2 arguments (3 given)"]
 
 def test_address_of_function():
     ffi = FFI()
@@ -1916,3 +1919,47 @@
     ffi.cdef("bool f(void);")
     lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
     assert lib.f() == 1
+
+def test_bool_in_cpp_2():
+    ffi = FFI()
+    ffi.cdef('int add(int a, int b);')
+    lib = verify(ffi, "test_bool_bug_cpp", '''
+        typedef bool _Bool;  /* there is a Windows header with this line */
+        int add(int a, int b)
+        {
+            return a + b;
+        }''', source_extension='.cpp')
+    c = lib.add(2, 3)
+    assert c == 5
+
+def test_struct_field_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[2]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+
+def test_function_arg_opaque():
+    py.test.skip("can currently declare a function with an opaque struct "
+                 "as argument, but AFAICT it's impossible to call it later")
+
+def test_function_returns_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a foo(int);")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_function_returns_opaque", "?")
+    assert str(e.value) == ("function foo: 'struct a' is used as result type,"
+                            " but is opaque")
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -133,6 +133,12 @@
         # You cannot assing character format codes as restype any longer
         raises(TypeError, setattr, f, "restype", "i")
 
+    def test_unicode_function_name(self):
+        f = dll[u'_testfunc_i_bhilfd']
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_int
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        assert result == 21
 
     def test_truncate_python_longs(self):
         f = dll._testfunc_i_bhilfd
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -159,7 +159,6 @@
         libraries=rtime.libraries
     )
     CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
@@ -233,7 +232,6 @@
 HAS_CLOCK_MONOTONIC = cConfig.CLOCK_MONOTONIC is not None
 HAS_MONOTONIC = (_WIN or _MACOSX or
                  (HAS_CLOCK_GETTIME and (HAS_CLOCK_HIGHRES or HAS_CLOCK_MONOTONIC)))
-clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
 
@@ -1030,7 +1028,10 @@
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
                 if rffi.cast(lltype.Signed, ret) != -1:
-                    cpu_time = float(tms.c_tms_utime + tms.c_tms_stime)
+                    cpu_time = float(rffi.cast(lltype.Signed,
+                                               tms.c_tms_utime) +
+                                     rffi.cast(lltype.Signed,
+                                               tms.c_tms_stime))
                     if w_info is not None:
                         _setinfo(space, w_info, "times()",
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
@@ -1038,7 +1039,7 @@
                     return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
         return clock(space)
 
-_clock = external('clock', [], clock_t)
+_clock = external('clock', [], rposix.CLOCK_T)
 def clock(space, w_info=None):
     """clock() -> floating point number
 
@@ -1052,7 +1053,7 @@
             pass
     value = _clock()
     # Is this casting correct?
-    if value == rffi.cast(clock_t, -1):
+    if intmask(value) == intmask(rffi.cast(rposix.CLOCK_T, -1)):
         raise oefmt(space.w_RuntimeError,
                     "the processor time used is not available or its value"
                     "cannot be represented")
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -434,4 +434,5 @@
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
 FakeObjSpace.sys.dlopenflags = 123
+FakeObjSpace.sys.track_resources = False
 FakeObjSpace.builtin = FakeModule()
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -930,6 +930,7 @@
         abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj))
 
 def type_get_dict(space, w_cls):
+    w_cls = _check(space, w_cls)
     from pypy.objspace.std.dictproxyobject import W_DictProxyObject
     w_dict = w_cls.getdict(space)
     if w_dict is None:
@@ -1287,8 +1288,8 @@
     cycle.append(candidate)
     cycle.reverse()
     names = [cls.getname(space) for cls in cycle]
-    raise OperationError(space.w_TypeError, space.wrap(
-        u"cycle among base classes: " + u' < '.join(names)))
+    raise oefmt(space.w_TypeError,
+                "cycle among base classes: %s", ' < '.join(names))
 
 
 class TypeCache(SpaceCache):
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -143,3 +143,5 @@
     def is_w(self, obj1, obj2):
         return obj1 is obj2
 
+    def setitem(self, obj, key, value):
+        obj[key] = value
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-# hypothesis is used for test generation on untranslated jit tests
+# hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -401,6 +401,9 @@
 class __extend__(pairtype(SomeString, SomeTuple),
                  pairtype(SomeUnicodeString, SomeTuple)):
     def mod((s_string, s_tuple)):
+        if not s_string.is_constant():
+            raise AnnotatorError("string formatting requires a constant "
+                                 "string/unicode on the left of '%'")
         is_string = isinstance(s_string, SomeString)
         is_unicode = isinstance(s_string, SomeUnicodeString)
         assert is_string or is_unicode
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4623,6 +4623,14 @@
         a = self.RPythonAnnotator()
         a.build_types(main, [int])
 
+    def test_string_mod_nonconstant(self):
+        def f(x):
+            return x % 5
+        a = self.RPythonAnnotator()
+        e = py.test.raises(AnnotatorError, a.build_types, f, [str])
+        assert ('string formatting requires a constant string/unicode'
+                in str(e.value))
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -883,6 +883,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                               self.cpu.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -280,7 +280,7 @@
     concrete_type = '\x00'
 
     def __init__(self, basesize, itemsize, lendescr, flag, is_pure=False, concrete_type='\x00'):
-        self.basesize = basesize
+        self.basesize = basesize    # this includes +1 for STR
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
@@ -676,7 +676,7 @@
 
 def unpack_arraydescr(arraydescr):
     assert isinstance(arraydescr, ArrayDescr)
-    ofs = arraydescr.basesize
+    ofs = arraydescr.basesize    # this includes +1 for STR
     size = arraydescr.itemsize
     sign = arraydescr.is_item_signed()
     return size, ofs, sign
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -293,6 +293,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_load_or_indexed(op, op.getarg(0), op.getarg(1),
                                          itemsize, itemsize, basesize, NOT_SIGNED)
         elif opnum == rop.UNICODEGETITEM:
@@ -304,6 +305,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_store_or_indexed(op, op.getarg(0), op.getarg(1), op.getarg(2),
                                          itemsize, itemsize, basesize)
         elif opnum == rop.UNICODESETITEM:
diff --git a/rpython/jit/backend/llsupport/symbolic.py b/rpython/jit/backend/llsupport/symbolic.py
--- a/rpython/jit/backend/llsupport/symbolic.py
+++ b/rpython/jit/backend/llsupport/symbolic.py
@@ -29,7 +29,7 @@
 def get_array_token(T, translate_support_code):
     # T can be an array or a var-sized structure
     if translate_support_code:
-        basesize = llmemory.sizeof(T, 0)
+        basesize = llmemory.sizeof(T, 0)     # this includes +1 for STR
         if isinstance(T, lltype.Struct):
             SUBARRAY = getattr(T, T._arrayfld)
             itemsize = llmemory.sizeof(SUBARRAY.OF)
@@ -57,6 +57,7 @@
             assert carray.length.size == WORD
             ofs_length = before_array_part + carray.length.offset
         basesize = before_array_part + carray.items.offset
+        basesize += T._hints.get('extra_item_after_alloc', 0)  # +1 for STR
         carrayitem = ll2ctypes.get_ctypes_type(T.OF)
         itemsize = ctypes.sizeof(carrayitem)
     return basesize, itemsize, ofs_length
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -435,8 +435,10 @@
 def test_bytearray_descr():
     c0 = GcCache(False)
     descr = get_array_descr(c0, rstr.STR)   # for bytearray
+    # note that we get a basesize that has 1 extra byte for the final null char
+    # (only for STR)
     assert descr.flag == FLAG_UNSIGNED
-    assert descr.basesize == struct.calcsize("PP")         # hash, length
+    assert descr.basesize == struct.calcsize("PP") + 1     # hash, length, extra
     assert descr.lendescr.offset == struct.calcsize("P")   # hash
     assert not descr.is_array_of_pointers()
 
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -647,6 +647,9 @@
         """)
 
     def test_rewrite_assembler_newstr_newunicode(self):
+        # note: strdescr.basesize already contains the extra final character,
+        # so that's why newstr(14) is rounded up to 'basesize+15' and not
+        # 'basesize+16'.
         self.check_rewrite("""
             [i2]
             p0 = newstr(14)
@@ -657,12 +660,12 @@
         """, """
             [i2]
             p0 = call_malloc_nursery(                                \
-                      %(strdescr.basesize + 16 * strdescr.itemsize + \
+                      %(strdescr.basesize + 15 * strdescr.itemsize + \
                         unicodedescr.basesize + 10 * unicodedescr.itemsize)d)
             gc_store(p0, 0,  %(strdescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p0, %(strlendescr.offset)s, 14, %(strlendescr.field_size)s)
             gc_store(p0, 0,  0, %(strhashdescr.field_size)s)
-            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
+            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 15 * strdescr.itemsize)d)
             gc_store(p1, 0,  %(unicodedescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p1, %(unicodelendescr.offset)s, 10, %(unicodelendescr.field_size)s)
             gc_store(p1, 0,  0, %(unicodehashdescr.field_size)s)
@@ -1240,14 +1243,14 @@
         #               'i3 = gc_load_i(p0,i5,%(unicodedescr.itemsize)d)'],
         [True,  (4,),  'i3 = strgetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,1,'
-                       '%(strdescr.basesize)d,1)'],
+                       '%(strdescr.basesize-1)d,1)'],
         #[False, (4,),  'i3 = strgetitem(p0,i1)' '->'
-        #               'i5 = int_add(i1, %(strdescr.basesize)d);'
+        #               'i5 = int_add(i1, %(strdescr.basesize-1)d);'
         #               'i3 = gc_load_i(p0,i5,1)'],
         ## setitem str/unicode
         [True, (4,),  'i3 = strsetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,1,'
-                               '%(strdescr.basesize)d,1)'],
+                               '%(strdescr.basesize-1)d,1)'],
         [True, (2,4), 'i3 = unicodesetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,'
                                  '%(unicodedescr.itemsize)d,'
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -3,7 +3,7 @@
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
 from rpython.rlib.jit import promote, _get_virtualizable_token
-from rpython.rlib import jit_hooks, rposix
+from rpython.rlib import jit_hooks, rposix, rgc
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
 from rpython.jit.backend.detect_cpu import getcpuclass
@@ -11,7 +11,7 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 from rpython.config.config import ConfigError
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.rlib.rjitlog import rjitlog as jl
 
 
@@ -29,6 +29,7 @@
         # - floats neg and abs
         # - cast_int_to_float
         # - llexternal with macro=True
+        # - extra place for the zero after STR instances
 
         class BasicFrame(object):
             _virtualizable_ = ['i']
@@ -56,7 +57,7 @@
             return ("/home.py",0,0)
 
         jitdriver = JitDriver(greens = [],
-                              reds = ['total', 'frame', 'j'],
+                              reds = ['total', 'frame', 'prev_s', 'j'],
                               virtualizables = ['frame'],
                               get_location = get_location)
         def f(i, j):
@@ -68,9 +69,12 @@
             total = 0
             frame = Frame(i)
             j = float(j)
+            prev_s = rstr.mallocstr(16)
             while frame.i > 3:
-                jitdriver.can_enter_jit(frame=frame, total=total, j=j)
-                jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                jitdriver.can_enter_jit(frame=frame, total=total, j=j,
+                                        prev_s=prev_s)
+                jitdriver.jit_merge_point(frame=frame, total=total, j=j,
+                                          prev_s=prev_s)
                 _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
@@ -82,6 +86,11 @@
                 k = myabs1(myabs2(j))
                 if k - abs(j):  raise ValueError
                 if k - abs(-j): raise ValueError
+                s = rstr.mallocstr(16)
+                rgc.ll_write_final_null_char(s)
+                rgc.ll_write_final_null_char(prev_s)
+                if (frame.i & 3) == 0:
+                    prev_s = s
             return chr(total % 253)
         #
         class Virt2(object):
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -994,6 +994,7 @@
             basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                         self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             scale = 0
 
         self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale)
diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -2,48 +2,157 @@
 from rpython.rlib import jit
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rvmprof import cintf
+from rpython.rlib.rvmprof import cintf, vmprof_execute_code, register_code,\
+    register_code_object_class, _get_vmprof
 from rpython.jit.backend.x86.arch import WORD
 from rpython.jit.codewriter.policy import JitPolicy
 
+
 class BaseRVMProfTest(object):
-    def test_one(self):
-        py.test.skip("needs thread-locals in the JIT, which is only available "
-                     "after translation")
+
+    def setup_method(self, meth):
         visited = []
 
         def helper():
+            trace = []
             stack = cintf.vmprof_tl_stack.getraw()
-            if stack:
-                # not during tracing
-                visited.append(stack.c_value)
-            else:
-                visited.append(0)
+            while stack:
+                trace.append((stack.c_kind, stack.c_value))
+                stack = stack.c_next
+            visited.append(trace)
 
         llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
 
-        driver = jit.JitDriver(greens=[], reds='auto')
+        class CodeObj(object):
+            def __init__(self, name):
+                self.name = name
 
-        def f(n):
+        def get_code_fn(codes, code, arg, c):
+            return code
+
+        def get_name(code):
+            return "foo"
+
+        _get_vmprof().use_weaklist = False
+        register_code_object_class(CodeObj, get_name)
+
+        self.misc = visited, llfn, CodeObj, get_code_fn, get_name
+
+
+    def teardown_method(self, meth):
+        del _get_vmprof().use_weaklist
+
+
+    def test_simple(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
             i = 0
             while i < n:
-                driver.jit_merge_point()
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if code.name == "main":
+                    c = f(codes, codes[1], 1, c)
+                else:
+                    llfn()
+                    c -= 1
                 i += 1
-                llfn()
+            return c
 
-        class Hooks(jit.JitHookInterface):
-            def after_compile(self, debug_info):
-                self.raw_start = debug_info.asminfo.rawstart
-
-        hooks = Hooks()
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            return f(codes, codes[0], n, 8)
 
         null = lltype.nullptr(cintf.VMPROFSTACK)
-        cintf.vmprof_tl_stack.setraw(null)   # make it empty
-        self.meta_interp(f, [10], policy=JitPolicy(hooks))
-        v = set(visited)
-        assert 0 in v
-        v.remove(0)
-        assert len(v) == 1
-        assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
-        assert cintf.vmprof_tl_stack.getraw() == null
-        # ^^^ make sure we didn't leave anything dangling
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+
+
+    def test_leaving_with_exception(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        class MyExc(Exception):
+            def __init__(self, c):
+                self.c = c
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
+            i = 0
+            while i < n:
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if code.name == "main":
+                    try:
+                        f(codes, codes[1], 1, c)
+                    except MyExc as e:
+                        c = e.c
+                else:
+                    llfn()
+                    c -= 1
+                i += 1
+            raise MyExc(c)
+
+        def main(n):
+            codes = [CodeObj("main"), CodeObj("not main")]
+            for code in codes:
+                register_code(code, get_name)
+            try:
+                f(codes, codes[0], n, 8)
+            except MyExc as e:
+                return e.c
+
+        null = lltype.nullptr(cintf.VMPROFSTACK)
+        cintf.vmprof_tl_stack.setraw(null)
+        self.meta_interp(main, [30], inline=True)
+        assert visited[:3] == [[(1, 12), (1, 8)], [(1, 12), (1, 8)], [(1, 12), (1, 8)]]
+
+
+    def test_leaving_with_exception_in_blackhole(self):
+        visited, llfn, CodeObj, get_code_fn, get_name = self.misc
+        driver = jit.JitDriver(greens=['code'], reds=['c', 'i', 'n', 'codes'])
+
+        class MyExc(Exception):
+            def __init__(self, c):
+                self.c = c
+
+        @vmprof_execute_code("main", get_code_fn,
+                             _hack_update_stack_untranslated=True)
+        def f(codes, code, n, c):
+            i = 0
+            while True:
+                driver.jit_merge_point(code=code, c=c, i=i, codes=codes, n=n)
+                if i >= n:

From pypy.commits at gmail.com  Fri Aug 12 11:58:26 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Fri, 12 Aug 2016 08:58:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix BUILD_SET_UNPACK by changing iterator
 to iter instead of itervalues as w_item should never be a dict anyway
Message-ID: <57adf222.274fc20a.2cefb.eec0@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86173:f658ed1189a5
Date: 2016-08-12 17:57 +0200
http://bitbucket.org/pypy/pypy/changeset/f658ed1189a5/

Log:	Fix BUILD_SET_UNPACK by changing iterator to iter instead of
	itervalues as w_item should never be a dict anyway

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1375,10 +1375,11 @@
         for i in range(itemcount, 0, -1):
             w_item = self.peekvalue(i-1)
             # cannot use w_sum.update, w_item might not be a set
-            iterator = w_item.itervalues()
+            iterator = space.iter(w_item)
             while True:
-                w_value = iterator.next_value()
-                if w_value is None:
+                try:
+                    w_value = space.next(iterator)
+                except OperationError:
                     break
                 w_sum.add(w_value)
         while itemcount != 0:

From pypy.commits at gmail.com  Fri Aug 12 12:35:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 12 Aug 2016 09:35:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Remove code added by py2-specific branch
 'resource_warning'
Message-ID: <57adfaea.c186c20a.6dddb.f379@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86174:b00718188b59
Date: 2016-08-12 17:35 +0100
http://bitbucket.org/pypy/pypy/changeset/b00718188b59/

Log:	Remove code added by py2-specific branch 'resource_warning'

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1801,40 +1801,6 @@
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
         """)
 
-    def resource_warning(self, w_msg, w_tb):
-        self.appexec([w_msg, w_tb],
-                     """(msg, tb):
-            import sys
-            print >> sys.stderr, msg
-            if tb:
-                print >> sys.stderr, "Created at (most recent call last):"
-                print >> sys.stderr, tb
-        """)
-
-    def format_traceback(self):
-        # we need to disable track_resources before calling the traceback
-        # module. Else, it tries to open more files to format the traceback,
-        # the file constructor will call space.format_traceback etc., in an
-        # inifite recursion
-        flag = self.sys.track_resources
-        self.sys.track_resources = False
-        try:
-            return self.appexec([],
-                         """():
-                import sys, traceback
-                # the "1" is because we don't want to show THIS code
-                # object in the traceback
-                try:
-                    f = sys._getframe(1)
-                except ValueError:
-                    # this happens if you call format_traceback at the very beginning
-                    # of startup, when there is no bottom code object
-                    return '<no stacktrace available>'
-                return "".join(traceback.format_stack(f))
-            """)
-        finally:
-            self.sys.track_resources = flag
-
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -209,14 +209,6 @@
         self.check(['-c', 'pass'], {'PYTHONNOUSERSITE': '1'}, sys_argv=['-c'],
                    run_command='pass', **expected)
 
-    def test_track_resources(self, monkeypatch):
-        myflag = [False]
-        def pypy_set_track_resources(flag):
-            myflag[0] = flag
-        monkeypatch.setattr(sys, 'pypy_set_track_resources', pypy_set_track_resources, raising=False)
-        self.check(['-X', 'track-resources'], {}, sys_argv=[''], run_stdin=True)
-        assert myflag[0] == True
-
 class TestInteraction:
     """
     These tests require pexpect (UNIX-only).
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -134,7 +134,7 @@
         assert self.space.lookup(w_instance, "gobbledygook") is None
         w_instance = self.space.appexec([], """():
             class Lookup(object):
-                "bla" 
+                "bla"
             return Lookup()""")
         assert self.space.str_w(self.space.lookup(w_instance, "__doc__")) == "bla"
 
@@ -148,7 +148,7 @@
         assert is_callable(w_func)
         w_lambda_func = self.space.appexec([], "(): return lambda: True")
         assert is_callable(w_lambda_func)
-        
+
         w_instance = self.space.appexec([], """():
             class Call(object):
                 def __call__(self): pass
@@ -308,7 +308,7 @@
 
     def test_call_obj_args(self):
         from pypy.interpreter.argument import Arguments
-        
+
         space = self.space
 
         w_f = space.appexec([], """():
@@ -333,7 +333,7 @@
         assert w_x is w_9
         assert w_y is w_1
 
-        w_res = space.call_obj_args(w_a, w_9, Arguments(space, []))        
+        w_res = space.call_obj_args(w_a, w_9, Arguments(space, []))
         assert w_res is w_9
 
     def test_compare_by_iteration(self):
@@ -383,7 +383,7 @@
         assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('g')))
         assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('h')))
 
-class TestModuleMinimal: 
+class TestModuleMinimal:
     def test_sys_exists(self):
         assert self.space.sys
 
@@ -458,28 +458,3 @@
         space.finish()
         # assert that we reach this point without getting interrupted
         # by the OperationError(NameError)
-
-    def test_format_traceback(self):
-        from pypy.tool.pytest.objspace import maketestobjspace
-        from pypy.interpreter.gateway import interp2app
-        #
-        def format_traceback(space):
-            return space.format_traceback()
-        #
-        space = maketestobjspace()
-        w_format_traceback = space.wrap(interp2app(format_traceback))
-        w_tb = space.appexec([w_format_traceback], """(format_traceback):
-            def foo():
-                return bar()
-            def bar():
-                return format_traceback()
-            return foo()
-        """)
-        tb = space.str_w(w_tb)
-        expected = '\n'.join([
-            '  File "?", line 6, in anonymous',  # this is the appexec code object
-            '  File "?", line 3, in foo',
-            '  File "?", line 5, in bar',
-            ''
-        ])
-        assert tb == expected
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -49,8 +49,6 @@
         '_current_frames'       : 'currentframes._current_frames',
         'setrecursionlimit'     : 'vm.setrecursionlimit',
         'getrecursionlimit'     : 'vm.getrecursionlimit',
-        'pypy_set_track_resources' : 'vm.set_track_resources',
-        'pypy_get_track_resources' : 'vm.get_track_resources',
         'setcheckinterval'      : 'vm.setcheckinterval',
         'getcheckinterval'      : 'vm.getcheckinterval',
         'exc_info'              : 'vm.exc_info',
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -61,13 +61,6 @@
     """
     return space.wrap(space.sys.recursionlimit)
 
- at unwrap_spec(flag=bool)
-def set_track_resources(space, flag):
-    space.sys.track_resources = flag
-
-def get_track_resources(space):
-    return space.wrap(space.sys.track_resources)
-
 @unwrap_spec(interval=int)
 def setcheckinterval(space, interval):
     """Tell the Python interpreter to check for asynchronous events every
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -434,5 +434,4 @@
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
 FakeObjSpace.sys.dlopenflags = 123
-FakeObjSpace.sys.track_resources = False
 FakeObjSpace.builtin = FakeModule()

From pypy.commits at gmail.com  Fri Aug 12 12:53:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 09:53:25 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Next emulation (we'll see how
 far it makes sense to continue)
Message-ID: <57adff05.497bc20a.13214.f7a8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86175:9dbb62851b3f
Date: 2016-08-12 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/9dbb62851b3f/

Log:	Next emulation (we'll see how far it makes sense to continue)

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -236,3 +236,7 @@
 def emulate_modf(x):
     return (llop.revdb_modf(lltype.Float, x, 0),
             llop.revdb_modf(lltype.Float, x, 1))
+
+def emulate_frexp(x):
+    return (llop.revdb_frexp(lltype.Float, x, 0),
+            int(llop.revdb_frexp(lltype.Float, x, 1)))
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -587,6 +587,7 @@
     'revdb_strtod':         LLOp(sideeffects=False),
     'revdb_dtoa':           LLOp(sideeffects=False),
     'revdb_modf':           LLOp(sideeffects=False),
+    'revdb_frexp':          LLOp(sideeffects=False),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py
--- a/rpython/rtyper/lltypesystem/module/ll_math.py
+++ b/rpython/rtyper/lltypesystem/module/ll_math.py
@@ -185,6 +185,8 @@
         mantissa = x
         exponent = 0
     else:
+        if revdb.flag_io_disabled():
+            return revdb.emulate_frexp(x)
         exp_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
         try:
             mantissa = math_frexp(x, exp_p)
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -243,6 +243,13 @@
         r = (index == 0) ? _r0 : _r1;                                   \
     } while (0)
 
+#define OP_REVDB_FREXP(x, index, r)                                     \
+    do {                                                                \
+        double _r0; int _r1;                                            \
+        _r0 = frexp(x, &_r1);                                           \
+        r = (index == 0) ? _r0 : _r1;                                   \
+    } while (0)
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -52,6 +52,9 @@
                 valx, valy = math.modf(val)
                 revdb.send_output(rdtoa.dtoa(valx) + '\n')
                 revdb.send_output(rdtoa.dtoa(valy) + '\n')
+                xx, yy = math.frexp(val)
+                revdb.send_output(rdtoa.dtoa(xx) + '\n')
+                revdb.send_output('%d\n' % yy)
                 return
             else:
                 assert False
@@ -210,4 +213,4 @@
         group = ReplayProcessGroup(str(self.exename), self.rdbname)
         with stdout_capture() as buf:
             group.print_cmd('2.35')
-        assert buf.getvalue() == "0.35\n2.0\n"
+        assert buf.getvalue() == "0.35\n2.0\n0.5875\n2\n"

From pypy.commits at gmail.com  Fri Aug 12 12:59:04 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 12 Aug 2016 09:59:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix translation
Message-ID: <57ae0058.e129c20a.4ece9.f97a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86176:b1def8b63787
Date: 2016-08-12 17:58 +0100
http://bitbucket.org/pypy/pypy/changeset/b1def8b63787/

Log:	Fix translation

diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -267,7 +267,7 @@
         space = self.space
         if self.accept_str and space.isinstance_w(w_init, space.w_str):
             # special case to optimize strings passed to a "char *" argument
-            value = w_init.str_w(space)
+            value = space.bytes_w(w_init)
             keepalives[i] = value
             buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
             rffi.cast(rffi.CCHARPP, cdata)[0] = buf

From pypy.commits at gmail.com  Fri Aug 12 13:49:25 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 12 Aug 2016 10:49:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix bad change in b053ff5c2d6d
Message-ID: <57ae0c25.94a51c0a.a5438.4c86@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86177:3a81057901f3
Date: 2016-08-12 18:48 +0100
http://bitbucket.org/pypy/pypy/changeset/3a81057901f3/

Log:	Fix bad change in b053ff5c2d6d

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1288,8 +1288,9 @@
     cycle.append(candidate)
     cycle.reverse()
     names = [cls.getname(space) for cls in cycle]
-    raise oefmt(space.w_TypeError,
-                "cycle among base classes: %s", ' < '.join(names))
+    # Can't use oefmt() here, since names is a list of unicodes
+    raise OperationError(space.w_TypeError, space.newunicode(
+        u"cycle among base classes: " + u' < '.join(names)))
 
 
 class TypeCache(SpaceCache):

From pypy.commits at gmail.com  Fri Aug 12 15:00:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 12:00:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: Test and probable fix (thanks sbauman)
Message-ID: <57ae1cc3.151a1c0a.a5160.638c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86178:cd39a869a312
Date: 2016-08-12 20:59 +0200
http://bitbucket.org/pypy/pypy/changeset/cd39a869a312/

Log:	Test and probable fix (thanks sbauman)

diff --git a/rpython/rlib/rweakref.py b/rpython/rlib/rweakref.py
--- a/rpython/rlib/rweakref.py
+++ b/rpython/rlib/rweakref.py
@@ -142,7 +142,7 @@
 
     def compute_result_annotation(self, s_keyclass, s_valueclass):
         assert s_keyclass.is_constant()
-        s_key = self.bookkeeper.immutablevalue(s_keyclass.const())
+        s_key = self.bookkeeper.valueoftype(s_keyclass.const)
         return SomeWeakValueDict(
             s_key,
             _getclassdef(s_valueclass))
@@ -158,7 +158,7 @@
         bk = self.bookkeeper
         x = self.instance
         return SomeWeakValueDict(
-            bk.immutablevalue(x._keyclass()),
+            bk.valueoftype(x._keyclass),
             bk.getuniqueclassdef(x._valueclass))
 
 def _getclassdef(s_instance):
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -180,3 +180,36 @@
         RWeakValueDictionary(str, X).get("foobar")
         RWeakValueDictionary(int, Y).get(42)
     interpret(g, [])
+
+def test_key_instance():
+    class K(object):
+        pass
+    keys = [K(), K(), K()]
+
+    def g(d):
+        assert d.get(keys[3]) is None
+        x1 = X(); x2 = X(); x3 = X()
+        d.set(keys[0], x1)
+        d.set(keys[1], x2)
+        d.set(keys[2], x3)
+        assert d.get(keys[0]) is x1
+        assert d.get(keys[1]) is x2
+        assert d.get(keys[2]) is x3
+        assert d.get(keys[3]) is None
+        return x1, x3    # x2 dies
+    def f():
+        keys.append(K())
+        d = RWeakValueDictionary(K, X)
+        x1, x3 = g(d)
+        rgc.collect(); rgc.collect()
+        assert d.get(keys[0]) is x1
+        assert d.get(keys[1]) is None
+        assert d.get(keys[2]) is x3
+        assert d.get(keys[3]) is None
+        d.set(keys[0], None)
+        assert d.get(keys[0]) is None
+        assert d.get(keys[1]) is None
+        assert d.get(keys[2]) is x3
+        assert d.get(keys[3]) is None
+    f()
+    interpret(f, [])

From pypy.commits at gmail.com  Fri Aug 12 15:27:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 12:27:39 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Install a minimal __import__
 hook to use interactively. At least this
Message-ID: <57ae232b.11051c0a.eea9a.6834@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86179:423804b65c4c
Date: 2016-08-12 21:27 +0200
http://bitbucket.org/pypy/pypy/changeset/423804b65c4c/

Log:	Install a minimal __import__ hook to use interactively. At least
	this version doesn't try to look for a file in the current
	directory, e.g. if we do "import math" instead a package, which just
	fails with "Attempted to do I/O".

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -321,18 +321,52 @@
     revdb.send_output(s)
     revdb.send_output("\n")
 
+ at gateway.unwrap_spec(name='str0', level=int)
+def revdb_importhook(space, name, w_globals=None,
+                     w_locals=None, w_fromlist=None, level=-1):
+    # Incredibly simplified version of __import__, which only returns
+    # already-imported modules and doesn't call any custom import
+    # hooks.  Recognizes only absolute imports.  With a 'fromlist'
+    # argument that is a non-empty list, returns the module 'name3' if
+    # the 'name' argument is 'name1.name2.name3'.  With an empty or
+    # None 'fromlist' argument, returns the module 'name1' instead.
+    return space.appexec([space.wrap(name), w_fromlist or space.w_None,
+                          space.wrap(level), space.wrap(space.sys)],
+    """(name, fromlist, level, sys):
+        if level > 0:
+            raise ImportError("only absolute imports are "
+                               "supported in the debugger")
+        basename = name.split('.')[0]
+        try:
+            basemod = sys.modules[basename]
+            mod = sys.modules[name]
+        except KeyError:
+            raise ImportError("'%s' not found or not imported yet "
+                    "(the debugger can't import new modules, "
+                    "and only supports absolute imports)" % (name,))
+        if fromlist:
+            return mod
+        return basemod
+    """)
+
 @specialize.memo()
 def get_revdb_displayhook(space):
     return space.wrap(gateway.interp2app(revdb_displayhook))
 
+ at specialize.memo()
+def get_revdb_importhook(space):
+    return space.wrap(gateway.interp2app(revdb_importhook))
+
 
 def prepare_print_environment(space):
     assert not dbstate.standard_code
     w_revdb_output = space.wrap(W_RevDBOutput(space))
     w_displayhook = get_revdb_displayhook(space)
+    w_import = get_revdb_importhook(space)
     space.sys.setdictvalue(space, 'stdout', w_revdb_output)
     space.sys.setdictvalue(space, 'stderr', w_revdb_output)
     space.sys.setdictvalue(space, 'displayhook', w_displayhook)
+    space.builtin.setdictvalue(space, '__import__', w_import)
 
 def command_print(cmd, expression):
     frame = fetch_cur_frame()

From pypy.commits at gmail.com  Fri Aug 12 16:54:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 13:54:09 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Trying to support pressing
 Ctrl-C to interrupt the current operation
Message-ID: <57ae3771.497bc20a.13214.4441@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86180:2311c3b8a675
Date: 2016-08-12 22:44 +0200
http://bitbucket.org/pypy/pypy/changeset/2311c3b8a675/

Log:	Trying to support pressing Ctrl-C to interrupt the current operation

diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -35,59 +35,79 @@
         self.print_extra_pending_info = None
 
     def interact(self):
-        last_command = 'help'
-        previous_time = None
-        previous_thread = 0
+        self.last_command = 'help'
+        self.previous_time = None
+        self.previous_thread = 0
         while True:
+            prompt = self.print_lines_before_prompt()
+            try:
+                while True:
+                    cmdline = self.display_prompt(prompt)
+                    self.run_command(cmdline)
+                    prompt = self.print_lines_before_prompt()
+            except KeyboardInterrupt:
+                self.pgroup.recreate_subprocess(self.previous_time or 1)
+                self.last_command = ''
+                self.previous_thread = '?'
+                self.previous_time = '?'
+
+    def print_lines_before_prompt(self):
+        last_time = self.pgroup.get_current_time()
+        if last_time != self.previous_time:
+            print
+            if self.pgroup.get_current_thread() != self.previous_thread:
+                self.previous_thread = self.pgroup.get_current_thread()
+                if self.previous_thread == 0:
+                    print ('-------------------- in main thread #0 '
+                           '--------------------')
+                else:
+                    print ('-------------------- in non-main thread '
+                           '#%d --------------------' % (self.previous_thread,))
+            self.pgroup.update_watch_values()
             last_time = self.pgroup.get_current_time()
-            if last_time != previous_time:
-                print
-                if self.pgroup.get_current_thread() != previous_thread:
-                    previous_thread = self.pgroup.get_current_thread()
-                    if previous_thread == 0:
-                        print ('-------------------- in main thread #0 '
-                               '--------------------')
-                    else:
-                        print ('-------------------- in non-main thread '
-                               '#%d --------------------' % (previous_thread,))
-                self.pgroup.update_watch_values()
-                last_time = self.pgroup.get_current_time()
-            if self.print_extra_pending_info:
-                print self.print_extra_pending_info
-                self.print_extra_pending_info = None
-            if last_time != previous_time:
-                self.pgroup.show_backtrace(complete=0)
-                previous_time = last_time
+        if self.print_extra_pending_info:
+            print self.print_extra_pending_info
+            self.print_extra_pending_info = None
+        if last_time != self.previous_time:
+            self.pgroup.show_backtrace(complete=0)
+            self.previous_time = last_time
+        prompt = '(%d)$ ' % last_time
+        return prompt
 
-            prompt = '(%d)$ ' % last_time
+    def display_prompt(self, prompt):
+        try:
+            cmdline = raw_input(prompt).strip()
+        except EOFError:
+            print
+            cmdline = 'quit'
+        if not cmdline:
+            cmdline = self.last_command
+        return cmdline
+
+    def run_command(self, cmdline):
+        match = r_cmdline.match(cmdline)
+        if not match:
+            return
+        self.last_command = cmdline
+        command, argument = match.groups()
+        try:
+            runner = getattr(self, 'command_' + command)
+        except AttributeError:
+            print >> sys.stderr, "no command '%s', try 'help'" % (command,)
+        else:
             try:
-                cmdline = raw_input(prompt).strip()
-            except EOFError:
-                print
-                cmdline = 'quit'
-            if not cmdline:
-                cmdline = last_command
-            match = r_cmdline.match(cmdline)
-            if not match:
-                continue
-            last_command = cmdline
-            command, argument = match.groups()
-            try:
-                runner = getattr(self, 'command_' + command)
-            except AttributeError:
-                print >> sys.stderr, "no command '%s', try 'help'" % (command,)
-            else:
-                try:
-                    runner(argument)
-                except Exception as e:
-                    traceback.print_exc()
-                    print >> sys.stderr
-                    print >> sys.stderr, 'Something went wrong.  You are now',
-                    print >> sys.stderr, 'in a pdb; press Ctrl-D to continue.'
-                    import pdb; pdb.post_mortem(sys.exc_info()[2])
-                    print >> sys.stderr
-                    print >> sys.stderr, 'You are back running %s.' % (
-                        sys.argv[0],)
+                runner(argument)
+            except KeyboardInterrupt:
+                raise
+            except Exception as e:
+                traceback.print_exc()
+                print >> sys.stderr
+                print >> sys.stderr, 'Something went wrong.  You are now',
+                print >> sys.stderr, 'in a pdb; press Ctrl-D to continue.'
+                import pdb; pdb.post_mortem(sys.exc_info()[2])
+                print >> sys.stderr
+                print >> sys.stderr, 'You are back running %s.' % (
+                    sys.argv[0],)
 
     def command_help(self, argument):
         """Display commands summary"""
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -132,11 +132,11 @@
         self.currently_created_objects = msg.arg2
         self.current_thread = msg.arg3
 
-    def clone(self):
+    def clone(self, activate=False):
         """Fork this subprocess.  Returns a new ReplayProcess() that is
         an identical copy.
         """
-        self.send(Message(CMD_FORK))
+        self.send(Message(CMD_FORK, int(activate)))
         s1, s2 = socket.socketpair()
         ancillary.send_fds(self.control_socket.fileno(), [s2.fileno()])
         s2.close()
@@ -459,7 +459,7 @@
         clone_me = self.paused[from_time]
         if self.active is not None:
             self.active.close()
-        self.active = clone_me.clone()
+        self.active = clone_me.clone(activate=True)
 
     def jump_in_time(self, target_time):
         """Jump in time at the given 'target_time'.
@@ -561,11 +561,13 @@
             self.active.send(Message(CMD_ATTACHID, nid, uid, int(watch_env)))
             self.active.expect_ready()
 
-    def recreate_subprocess(self):
-        # recreate a subprocess at the current time
-        time = self.get_current_time()
+    def recreate_subprocess(self, target_time=None):
+        # recreate a subprocess at the given time, or by default the
+        # current time
+        if target_time is None:
+            target_time = self.get_current_time()
         self.active = None
-        self.jump_in_time(time)
+        self.jump_in_time(target_time)
 
     def print_cmd(self, expression, nids=[]):
         """Print an expression.
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1249,7 +1249,7 @@
     exit(0);
 }
 
-static void command_fork(void)
+static void command_fork(int activate)
 {
     int child_sockfd;
     int child_pid;
@@ -1272,6 +1272,11 @@
         }
         rpy_rev_sockfd = child_sockfd;
 
+        /* The 'activate' flag of CMD_FORK tells if the child process
+           must die or not when receiving SIGINT.  Active children
+           die; inactive children (stored in 'pgroup.paused') don't. */
+        signal(SIGINT, activate ? SIG_DFL : SIG_IGN);
+
         /* Close and re-open the revdb log file in the child process.
            This is the simplest way I found to give 'rpy_rev_fileno'
            its own offset, independent from the parent.  It assumes
@@ -1422,7 +1427,7 @@
             switch (cmd.cmd) {
 
             case CMD_FORK:
-                command_fork();
+                command_fork(cmd.arg1);
                 break;
 
             case CMD_QUIT:
diff --git a/rpython/translator/revdb/test/ctrl_c.py b/rpython/translator/revdb/test/ctrl_c.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/test/ctrl_c.py
@@ -0,0 +1,43 @@
+import sys, os, thread, time, signal
+
+os.setpgid(0, 0)
+assert os.getpgrp() == os.getpid()
+
+
+sys.path[:] = sys.argv[1].split('\x7f')
+from rpython.translator.revdb.process import ReplayProcessGroup
+
+exename, rdbname = sys.argv[2:]
+group = ReplayProcessGroup(exename, rdbname)
+
+
+class MyInterrupt(Exception):
+    pass
+def my_signal(*args):
+    raise MyInterrupt
+prev_signal = signal.signal(signal.SIGINT, my_signal)
+
+def enable_timer():
+    def my_kill():
+        time.sleep(0.8)
+        print >> sys.stderr, "--<<< Sending CTRL-C >>>--"
+        os.killpg(os.getpid(), signal.SIGINT)
+    thread.start_new_thread(my_kill, ())
+
+all_ok = False
+try:
+    # this runs for ~9 seconds if uninterrupted
+    enable_timer()
+    group.print_cmd('very-long-loop')
+except MyInterrupt:
+    print >> sys.stderr, "very-long-loop interrupted, trying again"
+    group.recreate_subprocess(1)
+    try:
+        enable_timer()
+        group.print_cmd('very-long-loop')
+    except MyInterrupt:
+        print >> sys.stderr, "second interruption ok"
+        all_ok = True
+
+assert all_ok, "expected very-long-loop to be killed by SIGINT"
+print "all ok"
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -1,4 +1,4 @@
-import py, sys, math
+import py, sys, math, os, subprocess, time
 from cStringIO import StringIO
 from rpython.rlib import revdb, rdtoa
 from rpython.rlib.debug import debug_print, ll_assert
@@ -56,6 +56,14 @@
                 revdb.send_output(rdtoa.dtoa(xx) + '\n')
                 revdb.send_output('%d\n' % yy)
                 return
+            elif extra == 'very-long-loop':
+                i = 0
+                total = 0
+                while i < 2000000000:
+                    total += revdb.flag_io_disabled()
+                    i += 1
+                revdb.send_output(str(total))
+                return
             else:
                 assert False
             uid = revdb.get_unique_id(stuff)
@@ -214,3 +222,16 @@
         with stdout_capture() as buf:
             group.print_cmd('2.35')
         assert buf.getvalue() == "0.35\n2.0\n0.5875\n2\n"
+
+    def test_ctrl_c(self):
+        localdir = os.path.dirname(__file__)
+        args = [sys.executable, os.path.join(localdir, 'ctrl_c.py'),
+                '\x7f'.join(sys.path),
+                str(self.exename), self.rdbname]
+        t1 = time.time()
+        result = subprocess.check_output(args)
+        t2 = time.time()
+        print 'subprocess returned with captured stdout:\n%r' % (result,)
+        assert result == 'all ok\n'
+        # should take two times ~0.8 seconds if correctly interrupted
+        assert t2 - t1 < 3.0

From pypy.commits at gmail.com  Fri Aug 12 16:54:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 13:54:11 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Print something here
Message-ID: <57ae3773.81cb1c0a.93e5e.8134@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86181:c431addcce4e
Date: 2016-08-12 22:53 +0200
http://bitbucket.org/pypy/pypy/changeset/c431addcce4e/

Log:	Print something here

diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -46,7 +46,11 @@
                     self.run_command(cmdline)
                     prompt = self.print_lines_before_prompt()
             except KeyboardInterrupt:
-                self.pgroup.recreate_subprocess(self.previous_time or 1)
+                rtime = self.previous_time or 1
+                print
+                print 'KeyboardInterrupt: restoring state at time %d...' % (
+                    rtime,)
+                self.pgroup.recreate_subprocess(rtime)
                 self.last_command = ''
                 self.previous_thread = '?'
                 self.previous_time = '?'

From pypy.commits at gmail.com  Sat Aug 13 02:24:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 12 Aug 2016 23:24:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: For now,
 we can't specify both ``-O1`` and ``--platform=arm``: the first
Message-ID: <57aebd2c.28eac20a.ba98b.c3cc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86182:31c39adab9d0
Date: 2016-08-13 08:24 +0200
http://bitbucket.org/pypy/pypy/changeset/31c39adab9d0/

Log:	For now, we can't specify both ``-O1`` and ``--platform=arm``: the
	first option picks ``--gc=boehm`` and the second option picks
	``--gcrootfinder=shadowstack``, which are incompatible.

diff --git a/rpython/doc/arm.rst b/rpython/doc/arm.rst
--- a/rpython/doc/arm.rst
+++ b/rpython/doc/arm.rst
@@ -148,7 +148,7 @@
 
 ::
 
-  pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O1 --platform=arm target.py
+  pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O2 --platform=arm target.py
 
 If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``.
 

From pypy.commits at gmail.com  Sat Aug 13 03:20:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 13 Aug 2016 00:20:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document two branches
Message-ID: <57aeca2e.8628c20a.17598.d0f8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86183:888cb191638e
Date: 2016-08-13 09:19 +0200
http://bitbucket.org/pypy/pypy/changeset/888cb191638e/

Log:	Document two branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -128,3 +128,13 @@
 .. branch: cpyext-realloc
 
 Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.

From pypy.commits at gmail.com  Sat Aug 13 13:55:37 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sat, 13 Aug 2016 10:55:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Change python environment in conftest to
 python3.5
Message-ID: <57af5f19.c1e31c0a.3cb07.d707@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86184:6d502390ba4f
Date: 2016-08-13 19:54 +0200
http://bitbucket.org/pypy/pypy/changeset/6d502390ba4f/

Log:	Change python environment in conftest to python3.5

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -5,7 +5,7 @@
     # some tests fail otherwise
     sys.setrecursionlimit(2000)
 
-PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind('python3')
+PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind('python3.5')
 if PYTHON3 is not None:
     PYTHON3 = str(PYTHON3)
 

From pypy.commits at gmail.com  Sat Aug 13 15:00:52 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 13 Aug 2016 12:00:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: define _GUN_SOURCE before any other
 includes
Message-ID: <57af6e64.c75dc20a.853ec.b6f2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86185:d468895a18b8
Date: 2016-08-13 21:59 +0300
http://bitbucket.org/pypy/pypy/changeset/d468895a18b8/

Log:	define _GUN_SOURCE before any other includes

diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -2,6 +2,9 @@
 #define Py_PYTHON_H
 
 /* Compat stuff */
+#ifdef __GNUC__
+#define _GNU_SOURCE 1
+#endif
 #ifndef _WIN32
 # include <inttypes.h>
 # include <stdint.h>
@@ -52,7 +55,6 @@
 #ifndef DL_IMPORT
 #       define DL_IMPORT(RTYPE) RTYPE
 #endif
-
 #include <stdlib.h>
 
 #ifndef _WIN32

From pypy.commits at gmail.com  Sun Aug 14 03:11:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 14 Aug 2016 00:11:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document py2-mappingproxy
Message-ID: <57b019a0.28eac20a.ba98b.4f8f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86186:91db1a9b7bfd
Date: 2016-08-14 09:10 +0200
http://bitbucket.org/pypy/pypy/changeset/91db1a9b7bfd/

Log:	Document py2-mappingproxy

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -138,3 +138,9 @@
 Improved vmprof support: now tries hard to not miss any Python-level
 frame in the captured stacks, even if there is the metainterp or
 blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).

From pypy.commits at gmail.com  Sun Aug 14 10:17:13 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sun, 14 Aug 2016 07:17:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Undo change of conftest (load python3
 again), needs more fixes later
Message-ID: <57b07d69.2916c20a.26156.ceb4@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86187:d564da75d49d
Date: 2016-08-14 16:16 +0200
http://bitbucket.org/pypy/pypy/changeset/d564da75d49d/

Log:	Undo change of conftest (load python3 again), needs more fixes later

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -5,7 +5,7 @@
     # some tests fail otherwise
     sys.setrecursionlimit(2000)
 
-PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind('python3.5')
+PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind('python3')
 if PYTHON3 is not None:
     PYTHON3 = str(PYTHON3)
 

From pypy.commits at gmail.com  Sun Aug 14 16:12:14 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 14 Aug 2016 13:12:14 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: test showing current
 memoryview shortcomings
Message-ID: <57b0d09e.c186c20a.6dddb.3330@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86188:56370389b4ea
Date: 2016-08-14 21:17 +0300
http://bitbucket.org/pypy/pypy/changeset/56370389b4ea/

Log:	test showing current memoryview shortcomings

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3633,6 +3633,14 @@
         #assert a.base is data.__buffer__
         assert a.tostring() == 'abc'
 
+    def test_memoryview(self):
+        import numpy as np
+        x = np.array([1, 2, 3, 4, 5], dtype='i')
+        y = memoryview('abc')
+        assert y.format == 'B'
+        y = memoryview(x)
+        assert y.format == 'i'
+
     def test_fromstring(self):
         import sys
         from numpy import fromstring, dtype

From pypy.commits at gmail.com  Sun Aug 14 16:12:17 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 14 Aug 2016 13:12:17 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: add more passing tests
 for memoryview attributes
Message-ID: <57b0d0a1.c75dc20a.853ec.3835@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86190:ac8183b0c6c2
Date: 2016-08-14 22:08 +0300
http://bitbucket.org/pypy/pypy/changeset/ac8183b0c6c2/

Log:	add more passing tests for memoryview attributes

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3635,11 +3635,24 @@
 
     def test_memoryview(self):
         import numpy as np
+        import sys
+        if sys.version_info[:2] > (3, 2):
+            # In Python 3.3 the representation of empty shape, strides and sub-offsets
+            # is an empty tuple instead of None.
+            # http://docs.python.org/dev/whatsnew/3.3.html#api-changes
+            EMPTY = ()
+        else:
+            EMPTY = None
         x = np.array([1, 2, 3, 4, 5], dtype='i')
         y = memoryview('abc')
         assert y.format == 'B'
         y = memoryview(x)
         assert y.format == 'i'
+        assert y.shape == (5,)
+        assert y.ndim == 1
+        assert y.strides == (4,)
+        assert y.suboffsets == EMPTY
+        assert y.itemsize == 4
 
     def test_fromstring(self):
         import sys

From pypy.commits at gmail.com  Sun Aug 14 16:12:16 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 14 Aug 2016 13:12:16 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: move stride, format, ndim,
 itemsize, shape down to Buffer, override in ArrayBuffer
Message-ID: <57b0d0a0.82ddc20a.bde06.2948@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86189:0fcf7070dd43
Date: 2016-08-14 22:07 +0300
http://bitbucket.org/pypy/pypy/changeset/0fcf7070dd43/

Log:	move stride, format, ndim, itemsize, shape down to Buffer, override
	in ArrayBuffer

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -704,3 +704,20 @@
     def get_raw_address(self):
         from rpython.rtyper.lltypesystem import rffi
         return rffi.ptradd(self.impl.storage, self.impl.start)
+
+    def getformat(self):
+        return self.impl.dtype.char
+
+    def getitemsize(self):
+        return self.impl.dtype.elsize
+
+    def getndim(self):
+        return len(self.impl.shape)
+
+    def getshape(self):
+        return self.impl.shape
+
+    def getstrides(self):
+        return self.impl.strides
+
+
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -100,22 +100,22 @@
         return space.wrap(self.buf.getlength())
 
     def w_get_format(self, space):
-        return space.wrap("B")
+        return space.wrap(self.buf.getformat())
 
     def w_get_itemsize(self, space):
-        return space.wrap(1)
+        return space.wrap(self.buf.getitemsize())
 
     def w_get_ndim(self, space):
-        return space.wrap(1)
+        return space.wrap(self.buf.getndim())
 
     def w_is_readonly(self, space):
         return space.wrap(self.buf.readonly)
 
     def w_get_shape(self, space):
-        return space.newtuple([space.wrap(self.getlength())])
+        return space.newtuple([space.wrap(x) for x in self.buf.getshape()])
 
     def w_get_strides(self, space):
-        return space.newtuple([space.wrap(1)])
+        return space.newtuple([space.wrap(x) for x in self.buf.getstrides()])
 
     def w_get_suboffsets(self, space):
         # I've never seen anyone filling this field
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -59,6 +59,20 @@
     def get_raw_address(self):
         raise ValueError("no raw buffer")
 
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
 
 class StringBuffer(Buffer):
     __slots__ = ['value']

From pypy.commits at gmail.com  Mon Aug 15 05:17:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 15 Aug 2016 02:17:02 -0700 (PDT)
Subject: [pypy-commit] cffi default: Give an error when subtracting two
 pointers and the division's result is
Message-ID: <57b1888e.09afc20a.90904.e4f7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2737:763505916552
Date: 2016-08-15 11:03 +0200
http://bitbucket.org/cffi/cffi/changeset/763505916552/

Log:	Give an error when subtracting two pointers and the division's
	result is not exact (in gcc, we get nonsense, so it means it is
	undefined behavior for C, which is best handled by raising in cffi)

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2355,8 +2355,16 @@
             return NULL;
         }
         itemsize = ct->ct_itemdescr->ct_size;
-        if (itemsize <= 0) itemsize = 1;
-        diff = (cdv->c_data - cdw->c_data) / itemsize;
+        diff = cdv->c_data - cdw->c_data;
+        if (itemsize > 1) {
+            if (diff % itemsize) {
+                PyErr_SetString(PyExc_ValueError,
+                     "pointer subtraction: the distance between the two "
+                     "pointers is not a multiple of the item size");
+                return NULL;
+            }
+            diff = diff / itemsize;
+        }
 #if PY_MAJOR_VERSION < 3
         return PyInt_FromSsize_t(diff);
 #else
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -587,6 +587,19 @@
     e = py.test.raises(TypeError, "q - a")
     assert str(e.value) == "cannot subtract cdata 'short *' and cdata 'int *'"
 
+def test_ptr_sub_unaligned():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    a = cast(BIntPtr, 1240)
+    for bi in range(1430, 1438):
+        b = cast(BIntPtr, bi)
+        if ((bi - 1240) % size_of_int()) == 0:
+            assert b - a == (bi - 1240) // size_of_int()
+            assert a - b == (1240 - bi) // size_of_int()
+        else:
+            py.test.raises(ValueError, "b - a")
+            py.test.raises(ValueError, "a - b")
+
 def test_cast_primitive_from_cdata():
     p = new_primitive_type("int")
     n = cast(p, cast(p, -42))

From pypy.commits at gmail.com  Mon Aug 15 05:17:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 15 Aug 2016 02:17:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: cffi/763505916552: Give an error when
 subtracting two pointers and the
Message-ID: <57b188b0.47cbc20a.b54cb.e2be@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86195:b3b9882a5be7
Date: 2016-08-15 11:07 +0200
http://bitbucket.org/pypy/pypy/changeset/b3b9882a5be7/

Log:	cffi/763505916552: Give an error when subtracting two pointers and
	the division's result is not exact (in gcc, we get nonsense, so it
	means it is undefined behavior for C, which is best handled by
	raising in cffi)

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -310,11 +310,15 @@
                             self.ctype.name, ct.name)
             #
             itemsize = ct.ctitem.size
-            if itemsize <= 0:
-                itemsize = 1
             with self as ptr1, w_other as ptr2:
                 diff = (rffi.cast(lltype.Signed, ptr1) -
-                        rffi.cast(lltype.Signed, ptr2)) // itemsize
+                        rffi.cast(lltype.Signed, ptr2))
+            if itemsize > 1:
+                if diff % itemsize:
+                    raise oefmt(space.w_ValueError,
+                        "pointer subtraction: the distance between the two "
+                        "pointers is not a multiple of the item size")
+                diff //= itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -576,6 +576,19 @@
     e = py.test.raises(TypeError, "q - a")
     assert str(e.value) == "cannot subtract cdata 'short *' and cdata 'int *'"
 
+def test_ptr_sub_unaligned():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    a = cast(BIntPtr, 1240)
+    for bi in range(1430, 1438):
+        b = cast(BIntPtr, bi)
+        if ((bi - 1240) % size_of_int()) == 0:
+            assert b - a == (bi - 1240) // size_of_int()
+            assert a - b == (1240 - bi) // size_of_int()
+        else:
+            py.test.raises(ValueError, "b - a")
+            py.test.raises(ValueError, "a - b")
+
 def test_cast_primitive_from_cdata():
     p = new_primitive_type("int")
     n = cast(p, cast(p, -42))

From pypy.commits at gmail.com  Mon Aug 15 06:17:37 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 15 Aug 2016 03:17:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Merge with py3k
Message-ID: <57b196c1.8bc71c0a.8c8f4.45b9@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86196:360908aa059a
Date: 2016-08-15 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/360908aa059a/

Log:	Merge with py3k

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1801,40 +1801,6 @@
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
         """)
 
-    def resource_warning(self, w_msg, w_tb):
-        self.appexec([w_msg, w_tb],
-                     """(msg, tb):
-            import sys
-            print >> sys.stderr, msg
-            if tb:
-                print >> sys.stderr, "Created at (most recent call last):"
-                print >> sys.stderr, tb
-        """)
-
-    def format_traceback(self):
-        # we need to disable track_resources before calling the traceback
-        # module. Else, it tries to open more files to format the traceback,
-        # the file constructor will call space.format_traceback etc., in an
-        # inifite recursion
-        flag = self.sys.track_resources
-        self.sys.track_resources = False
-        try:
-            return self.appexec([],
-                         """():
-                import sys, traceback
-                # the "1" is because we don't want to show THIS code
-                # object in the traceback
-                try:
-                    f = sys._getframe(1)
-                except ValueError:
-                    # this happens if you call format_traceback at the very beginning
-                    # of startup, when there is no bottom code object
-                    return '<no stacktrace available>'
-                return "".join(traceback.format_stack(f))
-            """)
-        finally:
-            self.sys.track_resources = flag
-
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -209,14 +209,6 @@
         self.check(['-c', 'pass'], {'PYTHONNOUSERSITE': '1'}, sys_argv=['-c'],
                    run_command='pass', **expected)
 
-    def test_track_resources(self, monkeypatch):
-        myflag = [False]
-        def pypy_set_track_resources(flag):
-            myflag[0] = flag
-        monkeypatch.setattr(sys, 'pypy_set_track_resources', pypy_set_track_resources, raising=False)
-        self.check(['-X', 'track-resources'], {}, sys_argv=[''], run_stdin=True)
-        assert myflag[0] == True
-
 class TestInteraction:
     """
     These tests require pexpect (UNIX-only).
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -134,7 +134,7 @@
         assert self.space.lookup(w_instance, "gobbledygook") is None
         w_instance = self.space.appexec([], """():
             class Lookup(object):
-                "bla" 
+                "bla"
             return Lookup()""")
         assert self.space.str_w(self.space.lookup(w_instance, "__doc__")) == "bla"
 
@@ -148,7 +148,7 @@
         assert is_callable(w_func)
         w_lambda_func = self.space.appexec([], "(): return lambda: True")
         assert is_callable(w_lambda_func)
-        
+
         w_instance = self.space.appexec([], """():
             class Call(object):
                 def __call__(self): pass
@@ -308,7 +308,7 @@
 
     def test_call_obj_args(self):
         from pypy.interpreter.argument import Arguments
-        
+
         space = self.space
 
         w_f = space.appexec([], """():
@@ -333,7 +333,7 @@
         assert w_x is w_9
         assert w_y is w_1
 
-        w_res = space.call_obj_args(w_a, w_9, Arguments(space, []))        
+        w_res = space.call_obj_args(w_a, w_9, Arguments(space, []))
         assert w_res is w_9
 
     def test_compare_by_iteration(self):
@@ -383,7 +383,7 @@
         assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('g')))
         assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('h')))
 
-class TestModuleMinimal: 
+class TestModuleMinimal:
     def test_sys_exists(self):
         assert self.space.sys
 
@@ -458,28 +458,3 @@
         space.finish()
         # assert that we reach this point without getting interrupted
         # by the OperationError(NameError)
-
-    def test_format_traceback(self):
-        from pypy.tool.pytest.objspace import maketestobjspace
-        from pypy.interpreter.gateway import interp2app
-        #
-        def format_traceback(space):
-            return space.format_traceback()
-        #
-        space = maketestobjspace()
-        w_format_traceback = space.wrap(interp2app(format_traceback))
-        w_tb = space.appexec([w_format_traceback], """(format_traceback):
-            def foo():
-                return bar()
-            def bar():
-                return format_traceback()
-            return foo()
-        """)
-        tb = space.str_w(w_tb)
-        expected = '\n'.join([
-            '  File "?", line 6, in anonymous',  # this is the appexec code object
-            '  File "?", line 3, in foo',
-            '  File "?", line 5, in bar',
-            ''
-        ])
-        assert tb == expected
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -267,7 +267,7 @@
         space = self.space
         if self.accept_str and space.isinstance_w(w_init, space.w_str):
             # special case to optimize strings passed to a "char *" argument
-            value = w_init.str_w(space)
+            value = space.bytes_w(w_init)
             keepalives[i] = value
             buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
             rffi.cast(rffi.CCHARPP, cdata)[0] = buf
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -49,8 +49,6 @@
         '_current_frames'       : 'currentframes._current_frames',
         'setrecursionlimit'     : 'vm.setrecursionlimit',
         'getrecursionlimit'     : 'vm.getrecursionlimit',
-        'pypy_set_track_resources' : 'vm.set_track_resources',
-        'pypy_get_track_resources' : 'vm.get_track_resources',
         'setcheckinterval'      : 'vm.setcheckinterval',
         'getcheckinterval'      : 'vm.getcheckinterval',
         'exc_info'              : 'vm.exc_info',
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -61,13 +61,6 @@
     """
     return space.wrap(space.sys.recursionlimit)
 
- at unwrap_spec(flag=bool)
-def set_track_resources(space, flag):
-    space.sys.track_resources = flag
-
-def get_track_resources(space):
-    return space.wrap(space.sys.track_resources)
-
 @unwrap_spec(interval=int)
 def setcheckinterval(space, interval):
     """Tell the Python interpreter to check for asynchronous events every
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -434,5 +434,4 @@
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
 FakeObjSpace.sys.dlopenflags = 123
-FakeObjSpace.sys.track_resources = False
 FakeObjSpace.builtin = FakeModule()
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1288,8 +1288,9 @@
     cycle.append(candidate)
     cycle.reverse()
     names = [cls.getname(space) for cls in cycle]
-    raise oefmt(space.w_TypeError,
-                "cycle among base classes: %s", ' < '.join(names))
+    # Can't use oefmt() here, since names is a list of unicodes
+    raise OperationError(space.w_TypeError, space.newunicode(
+        u"cycle among base classes: " + u' < '.join(names)))
 
 
 class TypeCache(SpaceCache):

From pypy.commits at gmail.com  Mon Aug 15 09:54:03 2016
From: pypy.commits at gmail.com (vext01)
Date: Mon, 15 Aug 2016 06:54:03 -0700 (PDT)
Subject: [pypy-commit] pypy w-xor-x: Only allocate writable pages,
 and supply functions to change protection mask.
Message-ID: <57b1c97b.c19d1c0a.2bf58.9911@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: w-xor-x
Changeset: r86197:e32e8a566374
Date: 2016-08-15 14:51 +0100
http://bitbucket.org/pypy/pypy/changeset/e32e8a566374/

Log:	Only allocate writable pages, and supply functions to change
	protection mask.

diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -155,6 +155,8 @@
     c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT,
                                    rffi.INT, off_t], PTR, macro=True,
                                    save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO)
+    c_mprotect, _ = external('mprotect',
+                             [PTR, size_t, rffi.INT], rffi.INT)
     # 'mmap' on linux32 is a macro that calls 'mmap64'
     _, c_munmap_safe = external('munmap', [PTR, size_t], rffi.INT)
     c_msync, _ = external('msync', [PTR, size_t, rffi.INT], rffi.INT,
@@ -707,12 +709,22 @@
 
     def alloc_hinted(hintp, map_size):
         flags = MAP_PRIVATE | MAP_ANONYMOUS
-        prot = PROT_EXEC | PROT_READ | PROT_WRITE
+        prot = PROT_READ | PROT_WRITE
         if we_are_translated():
             flags = NonConstant(flags)
             prot = NonConstant(prot)
         return c_mmap_safe(hintp, map_size, prot, flags, -1, 0)
 
+    def set_pages_executable(addr, size):
+        rv = c_mprotect(addr, size, PROT_EXEC | PROT_READ)
+        if rv < 0:
+            debug.fatalerror_notb("set_pages_executable failed")
+
+    def set_pages_writable(addr, size):
+        rv = c_mprotect(addr, size, PROT_WRITE | PROT_READ)
+        if rv < 0:
+            debug.fatalerror_notb("set_pages_executable failed")
+
     def clear_large_memory_chunk_aligned(addr, map_size):
         addr = rffi.cast(PTR, addr)
         flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS
@@ -951,6 +963,9 @@
         return res
     alloc._annenforceargs_ = (int,)
 
+    def set_pages_executable(addr, size):
+        pass # XXX not implemented on windows
+
     def free(ptr, map_size):
         VirtualFree_safe(ptr, 0, MEM_RELEASE)
 

From pypy.commits at gmail.com  Mon Aug 15 09:54:05 2016
From: pypy.commits at gmail.com (vext01)
Date: Mon, 15 Aug 2016 06:54:05 -0700 (PDT)
Subject: [pypy-commit] pypy w-xor-x: Make cpu_info() W^X compliant.
Message-ID: <57b1c97d.d4e41c0a.5dcb6.8fa1@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: w-xor-x
Changeset: r86198:228237a37ee6
Date: 2016-08-15 14:52 +0100
http://bitbucket.org/pypy/pypy/changeset/228237a37ee6/

Log:	Make cpu_info() W^X compliant.

diff --git a/rpython/jit/backend/x86/detect_feature.py b/rpython/jit/backend/x86/detect_feature.py
--- a/rpython/jit/backend/x86/detect_feature.py
+++ b/rpython/jit/backend/x86/detect_feature.py
@@ -1,17 +1,20 @@
 import sys
 import struct
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rmmap import alloc, free
+from rpython.rlib.rmmap import alloc, free, set_pages_executable
+
+CPU_INFO_SZ = 4096
 
 def cpu_info(instr):
-    data = alloc(4096)
+    data = alloc(CPU_INFO_SZ)
     pos = 0
     for c in instr:
         data[pos] = c
         pos += 1
+    set_pages_executable(data, CPU_INFO_SZ)
     fnptr = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Signed)), data)
     code = fnptr()
-    free(data, 4096)
+    free(data, CPU_INFO_SZ)
     return code
 
 def detect_sse2():

From pypy.commits at gmail.com  Mon Aug 15 10:37:31 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 15 Aug 2016 07:37:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add cffi requirement for 'own' tests
Message-ID: <57b1d3ab.d8011c0a.a8158.a347@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86199:2f57d12b8365
Date: 2016-08-15 15:36 +0100
http://bitbucket.org/pypy/pypy/changeset/2f57d12b8365/

Log:	Add cffi requirement for 'own' tests

diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
+cffi>=1.4.0
+
 # hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2

From pypy.commits at gmail.com  Mon Aug 15 11:43:46 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 15 Aug 2016 08:43:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57b1e332.c4ebc20a.96bf2.738d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86200:f2a780e47063
Date: 2016-08-15 16:43 +0100
http://bitbucket.org/pypy/pypy/changeset/f2a780e47063/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -128,3 +128,19 @@
 .. branch: cpyext-realloc
 
 Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -310,11 +310,15 @@
                             self.ctype.name, ct.name)
             #
             itemsize = ct.ctitem.size
-            if itemsize <= 0:
-                itemsize = 1
             with self as ptr1, w_other as ptr2:
                 diff = (rffi.cast(lltype.Signed, ptr1) -
-                        rffi.cast(lltype.Signed, ptr2)) // itemsize
+                        rffi.cast(lltype.Signed, ptr2))
+            if itemsize > 1:
+                if diff % itemsize:
+                    raise oefmt(space.w_ValueError,
+                        "pointer subtraction: the distance between the two "
+                        "pointers is not a multiple of the item size")
+                diff //= itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -576,6 +576,19 @@
     e = py.test.raises(TypeError, "q - a")
     assert str(e.value) == "cannot subtract cdata 'short *' and cdata 'int *'"
 
+def test_ptr_sub_unaligned():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    a = cast(BIntPtr, 1240)
+    for bi in range(1430, 1438):
+        b = cast(BIntPtr, bi)
+        if ((bi - 1240) % size_of_int()) == 0:
+            assert b - a == (bi - 1240) // size_of_int()
+            assert a - b == (1240 - bi) // size_of_int()
+        else:
+            py.test.raises(ValueError, "b - a")
+            py.test.raises(ValueError, "a - b")
+
 def test_cast_primitive_from_cdata():
     p = new_primitive_type("int")
     n = cast(p, cast(p, -42))
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -2,6 +2,9 @@
 #define Py_PYTHON_H
 
 /* Compat stuff */
+#ifdef __GNUC__
+#define _GNU_SOURCE 1
+#endif
 #ifndef _WIN32
 # include <inttypes.h>
 # include <stdint.h>
@@ -52,7 +55,6 @@
 #ifndef DL_IMPORT
 #       define DL_IMPORT(RTYPE) RTYPE
 #endif
-
 #include <stdlib.h>
 
 #ifndef _WIN32
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
+cffi>=1.4.0
+
 # hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2
diff --git a/rpython/doc/arm.rst b/rpython/doc/arm.rst
--- a/rpython/doc/arm.rst
+++ b/rpython/doc/arm.rst
@@ -148,7 +148,7 @@
 
 ::
 
-  pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O1 --platform=arm target.py
+  pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O2 --platform=arm target.py
 
 If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``.
 
diff --git a/rpython/rlib/rweakref.py b/rpython/rlib/rweakref.py
--- a/rpython/rlib/rweakref.py
+++ b/rpython/rlib/rweakref.py
@@ -142,7 +142,7 @@
 
     def compute_result_annotation(self, s_keyclass, s_valueclass):
         assert s_keyclass.is_constant()
-        s_key = self.bookkeeper.immutablevalue(s_keyclass.const())
+        s_key = self.bookkeeper.valueoftype(s_keyclass.const)
         return SomeWeakValueDict(
             s_key,
             _getclassdef(s_valueclass))
@@ -158,7 +158,7 @@
         bk = self.bookkeeper
         x = self.instance
         return SomeWeakValueDict(
-            bk.immutablevalue(x._keyclass()),
+            bk.valueoftype(x._keyclass),
             bk.getuniqueclassdef(x._valueclass))
 
 def _getclassdef(s_instance):
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -180,3 +180,36 @@
         RWeakValueDictionary(str, X).get("foobar")
         RWeakValueDictionary(int, Y).get(42)
     interpret(g, [])
+
+def test_key_instance():
+    class K(object):
+        pass
+    keys = [K(), K(), K()]
+
+    def g(d):
+        assert d.get(keys[3]) is None
+        x1 = X(); x2 = X(); x3 = X()
+        d.set(keys[0], x1)
+        d.set(keys[1], x2)
+        d.set(keys[2], x3)
+        assert d.get(keys[0]) is x1
+        assert d.get(keys[1]) is x2
+        assert d.get(keys[2]) is x3
+        assert d.get(keys[3]) is None
+        return x1, x3    # x2 dies
+    def f():
+        keys.append(K())
+        d = RWeakValueDictionary(K, X)
+        x1, x3 = g(d)
+        rgc.collect(); rgc.collect()
+        assert d.get(keys[0]) is x1
+        assert d.get(keys[1]) is None
+        assert d.get(keys[2]) is x3
+        assert d.get(keys[3]) is None
+        d.set(keys[0], None)
+        assert d.get(keys[0]) is None
+        assert d.get(keys[1]) is None
+        assert d.get(keys[2]) is x3
+        assert d.get(keys[3]) is None
+    f()
+    interpret(f, [])

From pypy.commits at gmail.com  Mon Aug 15 12:01:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 15 Aug 2016 09:01:35 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: The CFFI + RevDB talk abstract
Message-ID: <57b1e75f.c186c20a.6dddb.841e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5667:ac0a0e384922
Date: 2016-08-15 18:01 +0200
http://bitbucket.org/pypy/extradoc/changeset/ac0a0e384922/

Log:	The CFFI + RevDB talk abstract

diff --git a/talk/pyconza2016/cffi-revdb.rst b/talk/pyconza2016/cffi-revdb.rst
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/cffi-revdb.rst
@@ -0,0 +1,33 @@
+
+CFFI, calling C // RevDB, a new debugger
+========================================
+
+
+Abstract
+--------
+
+Two different topics:
+
+* CFFI: a simple way to call C code from your Python programs;
+
+* RevDB: an experimental "reverse debugger" for Python.
+
+The two topics have in common their existence thanks to PyPy, an
+alternative Python implementation in Python.  Both are interesting
+even if you are only using the regular CPython.
+
+*CFFI* is an alternative to using the standard CPython C API to extend
+Python (or other tools like Cython, SWIG or ctypes).  It was
+originally inspired by LuaJIT's FFI.  Like Cython, you declare C
+functions and compile that with a regular C compiler.  Unlike Cython,
+there is no special language: you manipulate C data structures and
+call C functions straight from Python.  I will show examples of how
+simple it is to call existing C code with CFFI.
+
+*RevDB* is a reverse debugger for Python, similar to UndoDB-GDB or LL
+for C.  You run your program once, in "record" mode; then you start
+the reverse-debugger on the log file.  It gives a pdb-like experience,
+but it is replaying your program exactly as it ran---and moreover you
+can now go backward as well as forward in time.  You also get
+"watchpoints", which are very useful to find when things change.  I
+will show how it works on small examples.

From pypy.commits at gmail.com  Mon Aug 15 12:31:11 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 15 Aug 2016 09:31:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: properly close category even if we
 don't have debug_prints enabled, to support
Message-ID: <57b1ee4f.a710c20a.cc582.9794@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86201:6a6545b4a915
Date: 2016-08-15 18:30 +0200
http://bitbucket.org/pypy/pypy/changeset/6a6545b4a915/

Log:	properly close category even if we don't have debug_prints enabled,
	to support running with PYPYLOG=-

diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -98,6 +98,7 @@
     def log_abort_loop(self, trace, memo=None):
         debug_start("jit-abort-log")
         if not have_debug_prints():
+            debug_stop("jit-abort-log")
             return
         inputargs, operations = self._unpack_trace(trace)
         logops = self._log_operations(inputargs, operations, ops_offset=None,

From pypy.commits at gmail.com  Mon Aug 15 13:01:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 15 Aug 2016 10:01:31 -0700 (PDT)
Subject: [pypy-commit] pypy rpython-deque: Building a subset of deques in
 RPython: really just lists, but
Message-ID: <57b1f56b.45d11c0a.31711.d222@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-deque
Changeset: r86202:718e6e3e8ece
Date: 2016-08-14 09:13 +0200
http://bitbucket.org/pypy/pypy/changeset/718e6e3e8ece/

Log:	Building a subset of deques in RPython: really just lists, but with
	a "gap" at the start which allows O(1) implementation of a few
	operations like 'lst.insert(0, x)' and 'del lst[:n]'.


From pypy.commits at gmail.com  Mon Aug 15 13:01:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 15 Aug 2016 10:01:33 -0700 (PDT)
Subject: [pypy-commit] pypy rpython-deque: Annotation
Message-ID: <57b1f56d.898b1c0a.724cf.d481@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-deque
Changeset: r86203:63bf58058a3c
Date: 2016-08-14 09:27 +0200
http://bitbucket.org/pypy/pypy/changeset/63bf58058a3c/

Log:	Annotation

diff --git a/rpython/annotator/listdef.py b/rpython/annotator/listdef.py
--- a/rpython/annotator/listdef.py
+++ b/rpython/annotator/listdef.py
@@ -12,6 +12,7 @@
 class ListItem(object):
     mutated = False    # True for lists mutated after creation
     resized = False    # True for lists resized after creation
+    deque_hinted = False # True for 'list_implemented_as_deque()'
     range_step = None  # the step -- only for lists only created by a range()
     dont_change_any_more = False   # set to True when too late for changes
     immutable = False  # for getattr out of _immutable_fields_ = ['attr[*]']
@@ -49,6 +50,13 @@
                 raise ListChangeUnallowed("resizing list")
             self.resized = True
 
+    def deque_hint(self):
+        if not self.deque_hinted:
+            if self.dont_change_any_more:
+                raise TooLateForChange
+            self.resize()
+            self.deque_hinted = True
+
     def setrangestep(self, step):
         if step != self.range_step:
             if self.dont_change_any_more:
@@ -79,6 +87,8 @@
                 self.mutate()
             if other.resized:
                 self.resize()
+            if other.deque_hinted:
+                self.deque_hint()
             if other.range_step != self.range_step:
                 self.setrangestep(self._step_map[type(self.range_step),
                                                  type(other.range_step)])
@@ -173,9 +183,10 @@
         self.listitem.merge(newlistitem)
 
     def __repr__(self):
-        return '<[%r]%s%s%s%s>' % (self.listitem.s_value,
+        return '<[%r]%s%s%s%s%s>' % (self.listitem.s_value,
                                self.listitem.mutated and 'm' or '',
                                self.listitem.resized and 'r' or '',
+                               self.listitem.deque_hinted and 'd' or '',
                                self.listitem.immutable and 'I' or '',
                                self.listitem.must_not_resize and '!R' or '')
 
@@ -186,6 +197,10 @@
         self.listitem.mutate()
         self.listitem.resize()
 
+    def deque_hint(self):
+        self.resize()
+        self.listitem.deque_hint()
+
     def never_resize(self):
         if self.listitem.resized:
             raise ListChangeUnallowed("list already resized")
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4631,6 +4631,20 @@
         assert ('string formatting requires a constant string/unicode'
                 in str(e.value))
 
+    def test_list_implemented_as_deque(self):
+        def f(x):
+            if x > 5:
+                l = []
+            else:
+                l = []
+                objectmodel.list_implemented_as_deque(l)
+            return l
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [int])
+        assert s.listdef.listitem.mutated
+        assert s.listdef.listitem.resized
+        assert s.listdef.listitem.deque_hinted
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -431,6 +431,27 @@
         hop.exception_is_here()
         hop.gendirectcall(r_list.LIST._ll_resize_hint, v_list, v_sizehint)
 
+def list_implemented_as_deque(l):
+    """Hint that the list 'l' should use an implementation that allows
+    a "gap" at the start, which allows O(1) implementation of a few
+    operations like 'lst.insert(0, x)' and 'del lst[:n]'.
+    """
+
+class Entry(ExtRegistryEntry):
+    _about_ = list_implemented_as_deque
+
+    def compute_result_annotation(self, s_l):
+        from rpython.annotator import model as annmodel
+        if annmodel.s_None.contains(s_l):
+            return   # first argument is only None so far, but we
+                     # expect a generalization later
+        if not isinstance(s_l, annmodel.SomeList):
+            raise annmodel.AnnotatorError("Argument must be a list")
+        s_l.listdef.deque_hint()
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+
 # ____________________________________________________________
 #
 # id-like functions.  The idea is that calling hash() or id() is not

From pypy.commits at gmail.com  Mon Aug 15 13:01:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 15 Aug 2016 10:01:36 -0700 (PDT)
Subject: [pypy-commit] pypy rpython-deque: Add everywhere items+start
 instead of only items for all lists
Message-ID: <57b1f570.a710c20a.cc582.a1c0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-deque
Changeset: r86204:dcf07c3d78cf
Date: 2016-08-15 19:00 +0200
http://bitbucket.org/pypy/pypy/changeset/dcf07c3d78cf/

Log:	Add everywhere items+start instead of only items for all lists

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -839,7 +839,8 @@
         res = LIST.ll_newlist(dic.num_items)
         entries = dic.entries
         dlen = len(entries)
-        items = res.ll_items()
+        items, start = res.ll_items_start()
+        assert start == 0
         i = 0
         p = 0
         while i < dlen:
diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py
--- a/rpython/rtyper/lltypesystem/rlist.py
+++ b/rpython/rtyper/lltypesystem/rlist.py
@@ -17,6 +17,7 @@
 #  Concrete implementation of RPython lists:
 #
 #    struct list {
+#        int start;  // optional, default to 0
 #        int length;
 #        items_array *items;
 #    }
@@ -67,7 +68,7 @@
                                  "ll_newlist": ll_fixed_newlist,
                                  "ll_newemptylist": ll_fixed_newemptylist,
                                  "ll_length": ll_fixed_length,
-                                 "ll_items": ll_fixed_items,
+                                 "ll_items_start": ll_fixed_items_start,
                                  "ITEM": ITEM,
                                  "ll_getitem_fast": ll_fixed_getitem_fast,
                                  "ll_setitem_fast": ll_fixed_setitem_fast,
@@ -94,6 +95,11 @@
             ITEM = self.item_repr.lowleveltype
             ITEMARRAY = self.get_itemarray_lowleveltype()
             # XXX we might think of turning length stuff into Unsigned
+            extra = []
+            _ll_items_start = ll_items_0
+            if getattr(self.listitem, 'deque_hinted', False):
+                extra = [("start", Signed)]
+                _ll_items_start = ll_items_start
             self.LIST.become(GcStruct("list", ("length", Signed),
                                               ("items", Ptr(ITEMARRAY)),
                                       adtmeths = ADTIList({
@@ -101,7 +107,7 @@
                                           "ll_newlist_hint": ll_newlist_hint,
                                           "ll_newemptylist": ll_newemptylist,
                                           "ll_length": ll_length,
-                                          "ll_items": ll_items,
+                                          "ll_items_start": _ll_items_start,
                                           "ITEM": ITEM,
                                           "ll_getitem_fast": ll_getitem_fast,
                                           "ll_setitem_fast": ll_setitem_fast,
@@ -347,17 +353,22 @@
     return l.length
 ll_length.oopspec = 'list.len(l)'
 
-def ll_items(l):
-    return l.items
+def ll_items_0(l):
+    return l.items, 0
+
+def ll_items_start(l):
+    return l.items, l.start
 
 def ll_getitem_fast(l, index):
     ll_assert(index < l.length, "getitem out of bounds")
-    return l.ll_items()[index]
+    items, start = l.ll_items_start()
+    return items[start + index]
 ll_getitem_fast.oopspec = 'list.getitem(l, index)'
 
 def ll_setitem_fast(l, index, item):
     ll_assert(index < l.length, "setitem out of bounds")
-    l.ll_items()[index] = item
+    items, start = l.ll_items_start()
+    items[start + index] = item
 ll_setitem_fast.oopspec = 'list.setitem(l, index, item)'
 
 # fixed size versions
@@ -377,8 +388,8 @@
     return len(l)
 ll_fixed_length.oopspec = 'list.len(l)'
 
-def ll_fixed_items(l):
-    return l
+def ll_fixed_items_start(l):
+    return l, 0
 
 def ll_fixed_getitem_fast(l, index):
     ll_assert(index < len(l), "fixed getitem out of bounds")
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -1236,7 +1236,8 @@
         res = LIST.ll_newlist(dic.num_live_items)
         entries = dic.entries
         dlen = dic.num_ever_used_items
-        items = res.ll_items()
+        items, start = res.ll_items_start()
+        assert start == 0
         i = 0
         p = 0
         while i < dlen:
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -182,11 +182,18 @@
                             # where NULL is always valid: it is chr(0)
 
 
-    def _list_length_items(self, hop, v_lst, LIST):
+    def _list_length_items_start(self, hop, v_lst, LIST):
         LIST = LIST.TO
         v_length = hop.gendirectcall(LIST.ll_length, v_lst)
-        v_items = hop.gendirectcall(LIST.ll_items, v_lst)
-        return v_length, v_items
+        v_items = hop.gendirectcall(_ll_items_of, v_lst)
+        v_start = hop.gendirectcall(_ll_start_of, v_lst)
+        return v_length, v_items, v_start
+
+def _ll_items_of(l):
+    return l.ll_items_start()[0]
+def _ll_start_of(l):
+    return l.ll_items_start()[1]
+
 
 class StringRepr(BaseLLStringRepr, AbstractStringRepr):
     lowleveltype = Ptr(STR)
@@ -507,7 +514,7 @@
         return result
 
     @staticmethod
-    def ll_join(s, length, items):
+    def ll_join(s, length, items, start):
         s_chars = s.chars
         s_len = len(s_chars)
         num_items = length
@@ -517,7 +524,7 @@
         i = 0
         while i < num_items:
             try:
-                itemslen = ovfcheck(itemslen + len(items[i].chars))
+                itemslen = ovfcheck(itemslen + len(items[start + i].chars))
             except OverflowError:
                 raise MemoryError
             i += 1
@@ -528,14 +535,14 @@
         # a single '+' at the end is allowed to overflow: it gets
         # a negative result, and the gc will complain
         result = s.malloc(itemslen + seplen)
-        res_index = len(items[0].chars)
-        s.copy_contents(items[0], result, 0, 0, res_index)
+        res_index = len(items[start].chars)
+        s.copy_contents(items[start], result, 0, 0, res_index)
         i = 1
         while i < num_items:
             s.copy_contents(s, result, 0, res_index, s_len)
             res_index += s_len
-            lgt = len(items[i].chars)
-            s.copy_contents(items[i], result, 0, res_index, lgt)
+            lgt = len(items[start + i].chars)
+            s.copy_contents(items[start + i], result, 0, res_index, lgt)
             res_index += lgt
             i += 1
         return result
@@ -811,20 +818,20 @@
         return count
 
     @staticmethod
-    @signature(types.int(), types.any(), returns=types.any())
-    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
-        items, length))
-    def ll_join_strs(length, items):
+    @signature(types.int(), types.any(), types.int(), returns=types.any())
+    @jit.look_inside_iff(lambda length, items, start:
+                         jit.loop_unrolling_heuristic(items, length))
+    def ll_join_strs(length, items, start):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
-            return items[0]
+            return items[start]
 
         num_items = length
         itemslen = 0
         i = 0
         while i < num_items:
             try:
-                itemslen = ovfcheck(itemslen + len(items[i].chars))
+                itemslen = ovfcheck(itemslen + len(items[start + i].chars))
             except OverflowError:
                 raise MemoryError
             i += 1
@@ -838,18 +845,17 @@
         res_index = 0
         i = 0
         while i < num_items:
-            item_chars = items[i].chars
+            item_chars = items[start + i].chars
             item_len = len(item_chars)
-            copy_contents(items[i], result, 0, res_index, item_len)
+            copy_contents(items[start + i], result, 0, res_index, item_len)
             res_index += item_len
             i += 1
         return result
 
     @staticmethod
-    @jit.look_inside_iff(lambda length, chars, RES: jit.isconstant(length) and jit.isvirtual(chars))
-    def ll_join_chars(length, chars, RES):
-        # no need to optimize this, will be replaced by string builder
-        # at some point soon
+    @jit.look_inside_iff(lambda length, chars, start, RES:
+                         jit.isconstant(length) and jit.isvirtual(chars))
+    def ll_join_chars(length, chars, start, RES):
         num_chars = length
         if RES is StringRepr.lowleveltype:
             target = Char
@@ -861,7 +867,7 @@
         res_chars = result.chars
         i = 0
         while i < num_chars:
-            res_chars[i] = cast_primitive(target, chars[i])
+            res_chars[i] = cast_primitive(target, chars[start + i])
             i += 1
         return result
 
@@ -918,7 +924,8 @@
                     break
             i += 1
         res = LIST.ll_newlist(count)
-        items = res.ll_items()
+        items, start = res.ll_items_start()
+        assert start == 0
         i = 0
         j = 0
         resindex = 0
@@ -951,7 +958,8 @@
             pos = s.find(c, pos + markerlen, last)
             count += 1
         res = LIST.ll_newlist(count)
-        items = res.ll_items()
+        items, start = res.ll_items_start()
+        assert start == 0
         pos = 0
         count = 0
         pos = s.find(c, 0, last)
@@ -985,7 +993,8 @@
                     break
             i += 1
         res = LIST.ll_newlist(count)
-        items = res.ll_items()
+        items, start = res.ll_items_start()
+        assert start == 0
         i = strlen
         j = strlen
         resindex = count - 1
@@ -1018,7 +1027,8 @@
             pos = s.rfind(c, 0, pos - markerlen)
             count += 1
         res = LIST.ll_newlist(count)
-        items = res.ll_items()
+        items, start = res.ll_items_start()
+        assert start == 0
         pos = 0
         pos = len(s.chars)
         prev_pos = pos
@@ -1133,7 +1143,7 @@
 
     @staticmethod
     def ll_build_finish(builder):
-        return LLHelpers.ll_join_strs(len(builder), builder)
+        return LLHelpers.ll_join_strs(len(builder), builder, 0)
 
     @staticmethod
     @specialize.memo()
@@ -1210,14 +1220,16 @@
             hop.genop('setarrayitem', [vtemp, i, vchunk])
 
         hop.exception_cannot_occur()   # to ignore the ZeroDivisionError of '%'
-        return hop.gendirectcall(cls.ll_join_strs, size, vtemp)
+        c_zero = inputconst(Signed, 0)
+        return hop.gendirectcall(cls.ll_join_strs, size, vtemp, c_zero)
 
     @staticmethod
     @jit.dont_look_inside
     def ll_string2list(RESLIST, src):
         length = len(src.chars)
         lst = RESLIST.ll_newlist(length)
-        dst = lst.ll_items()
+        dst, start = lst.ll_items_start()
+        assert start == 0
         SRC = typeOf(src).TO     # STR or UNICODE
         DST = typeOf(dst).TO     # GcArray
         assert DST.OF is SRC.chars.OF
diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py
--- a/rpython/rtyper/rlist.py
+++ b/rpython/rtyper/rlist.py
@@ -555,8 +555,10 @@
 def ll_arraycopy(source, dest, source_start, dest_start, length):
     SRCTYPE = typeOf(source)
     # lltype
-    rgc.ll_arraycopy(source.ll_items(), dest.ll_items(),
-                     source_start, dest_start, length)
+    srcitems, srcstart = source.ll_items_start()
+    dstitems, dststart = dest.ll_items_start()
+    rgc.ll_arraycopy(srcitems, dstitems,
+                     srcstart + source_start, dststart + dest_start, length)
 
 
 def ll_copy(RESLIST, l):
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -203,7 +203,7 @@
         hop.exception_cannot_occur()
         return hop.gendirectcall(self.ll.ll_isalnum, v_str)
 
-    def _list_length_items(self, hop, v_lst, LIST):
+    def _list_length_items_start(self, hop, v_lst, LIST):
         """Return two Variables containing the length and items of a
         list. Need to be overriden because it is typesystem-specific."""
         raise NotImplementedError
@@ -219,7 +219,8 @@
         if not isinstance(r_lst, BaseListRepr):
             raise TyperError("string.join of non-list: %r" % r_lst)
         v_str, v_lst = hop.inputargs(rstr.repr, r_lst)
-        v_length, v_items = self._list_length_items(hop, v_lst, r_lst.lowleveltype)
+        v_length, v_items, v_start = self._list_length_items_start(hop, v_lst,
+                                                            r_lst.lowleveltype)
 
         if hop.args_s[0].is_constant() and hop.args_s[0].const == '':
             if r_lst.item_repr == rstr.repr:
@@ -228,16 +229,16 @@
                   r_lst.item_repr == unichar_repr):
                 v_tp = hop.inputconst(Void, self.lowleveltype)
                 return hop.gendirectcall(self.ll.ll_join_chars, v_length,
-                                         v_items, v_tp)
+                                         v_items, v_start, v_tp)
             else:
                 raise TyperError("''.join() of non-string list: %r" % r_lst)
-            return hop.gendirectcall(llfn, v_length, v_items)
+            return hop.gendirectcall(llfn, v_length, v_items, v_start)
         else:
             if r_lst.item_repr == rstr.repr:
                 llfn = self.ll.ll_join
             else:
                 raise TyperError("sep.join() of non-string list: %r" % r_lst)
-            return hop.gendirectcall(llfn, v_str, v_length, v_items)
+            return hop.gendirectcall(llfn, v_str, v_length, v_items, v_start)
 
     def rtype_method_splitlines(self, hop):
         rstr = hop.args_r[0].repr
diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py
--- a/rpython/rtyper/test/test_llinterp.py
+++ b/rpython/rtyper/test/test_llinterp.py
@@ -207,9 +207,11 @@
     def f():
         return [1,2,3]
     res = interpret(f,[])
-    assert len(res.ll_items()) == len([1,2,3])
+    items, start = res.ll_items_start()
+    assert start == 0
+    assert len(items) == len([1,2,3])
     for i in range(3):
-        assert res.ll_items()[i] == i+1
+        assert items[i] == i+1
 
 def test_list_itemops():
     def f(i):
@@ -250,10 +252,12 @@
         l.reverse()
         return l
     res = interpret(f,[])
-    assert len(res.ll_items()) == len([3,2,1])
+    items, start = res.ll_items_start()
+    assert start == 0
+    assert len(items) == len([3,2,1])
     print res
     for i in range(3):
-        assert res.ll_items()[i] == 3-i
+        assert items[i] == 3-i
 
 def test_list_pop():
     def f():
@@ -263,7 +267,9 @@
         l3 = l.pop(-1)
         return [l1,l2,l3]
     res = interpret(f,[])
-    assert len(res.ll_items()) == 3
+    items, start = res.ll_items_start()
+    assert start == 0
+    assert len(items) == 3
 
 def test_ovf():
     def f(x):
diff --git a/rpython/rtyper/test/tool.py b/rpython/rtyper/test/tool.py
--- a/rpython/rtyper/test/tool.py
+++ b/rpython/rtyper/test/tool.py
@@ -77,8 +77,8 @@
     @staticmethod
     def ll_to_list(l):
         r = []
-        items = l.ll_items()
-        for i in range(l.ll_length()):
+        items, start = l.ll_items_start()
+        for i in range(start, start + l.ll_length()):
             r.append(items[i])
         return r
 

From pypy.commits at gmail.com  Mon Aug 15 16:13:01 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 15 Aug 2016 13:13:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add asyncio test for asynchronous context
 managers
Message-ID: <57b2224d.531d1c0a.9874.14ea@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86205:90eef5573de8
Date: 2016-08-15 22:11 +0200
http://bitbucket.org/pypy/pypy/changeset/90eef5573de8/

Log:	Add asyncio test for asynchronous context managers

diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -18,3 +18,31 @@
         loop.run_until_complete(f())
         print("done with async loop")
         """
+    
+    def test_asynchronous_context_managers(self):
+        """
+import encodings.idna
+import asyncio
+
+class Corotest(object):
+    def __init__(self):
+        self.res = "-"
+    
+    async def coro(self, name, lock):
+        self.res += ' coro {}: waiting for lock -'.format(name)
+        async with lock:
+            self.res += ' coro {}: holding the lock -'.format(name)
+            await asyncio.sleep(1)
+            self.res += ' coro {}: releasing the lock -'.format(name)
+
+cor = Corotest()
+loop = asyncio.get_event_loop()
+lock = asyncio.Lock()
+coros = asyncio.gather(cor.coro(1, lock), cor.coro(2, lock))
+try:
+    loop.run_until_complete(coros)
+finally:
+    loop.close()
+
+assert cor.res == "- coro 1: waiting for lock - coro 1: holding the lock - coro 2: waiting for lock - coro 1: releasing the lock - coro 2: holding the lock - coro 2: releasing the lock -"
+        """

From pypy.commits at gmail.com  Mon Aug 15 19:44:01 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 15 Aug 2016 16:44:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Add usemodules declarations that py3k
 seems to require
Message-ID: <57b253c1.c62f1c0a.ea45f.4036@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86206:958c3593f88a
Date: 2016-08-16 00:43 +0100
http://bitbucket.org/pypy/pypy/changeset/958c3593f88a/

Log:	Add usemodules declarations that py3k seems to require

diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -79,7 +79,7 @@
 
 
 class AppTestRecompiler:
-    spaceconfig = dict(usemodules=['_cffi_backend', 'imp'])
+    spaceconfig = dict(usemodules=['_cffi_backend', 'imp', 'cpyext', 'struct'])
 
     def setup_class(cls):
         if cls.runappdirect:

From pypy.commits at gmail.com  Mon Aug 15 20:13:59 2016
From: pypy.commits at gmail.com (wlav)
Date: Mon, 15 Aug 2016 17:13:59 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: from Aditi: resolve std::string
 ctor overload with length
Message-ID: <57b25ac7.11331c0a.388e3.87ef@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86207:eb960a97982c
Date: 2016-08-15 17:03 -0700
http://bitbucket.org/pypy/pypy/changeset/eb960a97982c/

Log:	from Aditi: resolve std::string ctor overload with length

diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -17,10 +17,11 @@
     pass
 
 # overload priorities: lower is preferred
-priority = { 'void*'  : 100,
-             'void**' : 100,
-             'float'  :  30,
-             'double' :  10, }
+priority = { 'void*'         : 100,
+             'void**'        : 100,
+             'float'         :  30,
+             'double'        :  10,
+             'const string&' :   1, } # solves a specific string ctor overload
 
 from rpython.rlib.listsort import make_timsort_class
 CPPMethodBaseTimSort = make_timsort_class()

From pypy.commits at gmail.com  Mon Aug 15 20:14:01 2016
From: pypy.commits at gmail.com (wlav)
Date: Mon, 15 Aug 2016 17:14:01 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: from Aditi (modified): allow
 passing of strings with \0 in them
Message-ID: <57b25ac9.031dc20a.de9d1.14a4@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86208:7f130da8dc67
Date: 2016-08-15 17:09 -0700
http://bitbucket.org/pypy/pypy/changeset/7f130da8dc67/

Log:	from Aditi (modified): allow passing of strings with \0 in them

diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -158,11 +158,15 @@
     return _c_call_r(cppmethod, cppobject, nargs, args)
 _c_call_s = rffi.llexternal(
     "cppyy_call_s",
-    [C_METHOD, C_OBJECT, rffi.INT, rffi.VOIDP], rffi.CCHARP,
+    [C_METHOD, C_OBJECT, rffi.INT, rffi.VOIDP, rffi.INTP], rffi.CCHARP,
     releasegil=ts_call,
     compilation_info=backend.eci)
 def c_call_s(space, cppmethod, cppobject, nargs, args):
-    return _c_call_s(cppmethod, cppobject, nargs, args)
+    length = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    cstr = _c_call_s(cppmethod, cppobject, nargs, args, length)
+    cstr_len = int(length[0])
+    lltype.free(length, flavor='raw')
+    return cstr, cstr_len
 
 _c_constructor = rffi.llexternal(
     "cppyy_constructor",
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -146,7 +146,8 @@
             'call_d'       : ([c_method, c_object, c_int, c_voidp],   c_double),
 
             'call_r'       : ([c_method, c_object, c_int, c_voidp],   c_voidp),
-            'call_s'       : ([c_method, c_object, c_int, c_voidp],   c_ccharp),
+            # call_s actually takes an intp as last parameter, but this will do
+            'call_s'       : ([c_method, c_object, c_int, c_voidp, c_voidp],    c_ccharp),
 
             'constructor'  : ([c_method, c_object, c_int, c_voidp],   c_object),
             'call_o'       : ([c_method, c_object, c_int, c_voidp, c_type],     c_object),
@@ -336,8 +337,12 @@
     args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
     return _cdata_to_ptr(space, call_capi(space, 'call_r', args))
 def c_call_s(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
-    return call_capi(space, 'call_s', args)
+    length = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs), _Args(vp=length)]
+    cstr = call_capi(space, 'call_s', args)
+    cstr_len = int(length[0])
+    lltype.free(length, flavor='raw')
+    return cstr, cstr_len
 
 def c_constructor(space, cppmethod, cppobject, nargs, cargs):
     args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -195,8 +195,10 @@
 class StdStringExecutor(InstancePtrExecutor):
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
-        cstr_result = capi.c_call_s(space, cppmethod, cppthis, num_args, args)
-        return space.wrap(capi.charp2str_free(space, cstr_result))
+        cstr, cstr_len = capi.c_call_s(space, cppmethod, cppthis, num_args, args)
+        string = rffi.charpsize2str(cstr, cstr_len)
+        capi.c_free(rffi.cast(rffi.VOIDP, cstr))
+        return space.wrap(string)
 
     def execute_libffi(self, space, cif_descr, funcaddr, buffer):
         from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -59,7 +59,7 @@
     RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
     RPY_EXTERN
-    char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, int* length);
 
     RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
diff --git a/pypy/module/cppyy/include/cpp_cppyy.h b/pypy/module/cppyy/include/cpp_cppyy.h
--- a/pypy/module/cppyy/include/cpp_cppyy.h
+++ b/pypy/module/cppyy/include/cpp_cppyy.h
@@ -62,7 +62,7 @@
    Double_t     CallD( TCppMethod_t method, TCppObject_t self, void* args );
    LongDouble_t CallLD( TCppMethod_t method, TCppObject_t self, void* args );
    void*        CallR( TCppMethod_t method, TCppObject_t self, void* args );
-   Char_t*      CallS( TCppMethod_t method, TCppObject_t self, void* args );
+   Char_t*      CallS( TCppMethod_t method, TCppObject_t self, void* args, int* length );
    TCppObject_t CallConstructor( TCppMethod_t method, TCppType_t type, void* args );
    void         CallDestructor( TCppType_t type, TCppObject_t self );
    TCppObject_t CallO( TCppMethod_t method, TCppObject_t self, void* args, TCppType_t result_type );
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -113,10 +113,10 @@
 }
 
 static inline
-char* cppstring_to_cstring( const std::string& name ) {
-   char* name_char = (char*)malloc(name.size() + 1 );
-   strcpy( name_char, name.c_str() );
-   return name_char;
+char* cppstring_to_cstring( const std::string& cppstr ) {
+   char* cstr = (char*)malloc( cppstr.size() + 1 );
+   memcpy( cstr, cppstr.c_str(), cppstr.size() + 1 );
+   return cstr;
 }
 
 
@@ -478,12 +478,20 @@
    return nullptr;
 }
 
-Char_t* Cppyy::CallS( TCppMethod_t method, TCppObject_t self, void* args )
+Char_t* Cppyy::CallS(
+      TCppMethod_t method, TCppObject_t self, void* args, int* length )
 {
-   Char_t* s = nullptr;
-   if ( FastCall( method, args, (void*)self, &s ) )
-      return s;
-   return nullptr;
+   char* cstr = nullptr;
+   TClassRef cr("std::string");
+   std::string* cppresult = (std::string*)malloc( sizeof(std::string) );
+   if ( FastCall( method, args, self, (void*)cppresult ) ) {
+	  cstr = cppstring_to_cstring( *cppresult );
+      *length = cppresult->size();
+      cppresult->std::string::~string();
+   } else
+      *length = 0;
+   free( (void*)cppresult ); 
+   return cstr;
 }
 
 Cppyy::TCppObject_t Cppyy::CallConstructor(
@@ -1188,20 +1196,20 @@
     return (double)Cppyy::CallD(method, (void*)self, &parvec);
 }
 
-void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
+void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     std::vector<TParameter> parvec = vsargs_to_parvec(args, nargs);
     return (void*)Cppyy::CallR(method, (void*)self, &parvec);
 }
 
-char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
+char* cppyy_call_s(
+        cppyy_method_t method, cppyy_object_t self, int nargs, void* args, int* length) {
     std::vector<TParameter> parvec = vsargs_to_parvec(args, nargs);
-    return cppstring_to_cstring(Cppyy::CallS(method, (void*)self, &parvec));
+    return Cppyy::CallS(method, (void*)self, &parvec, length);
 }
 
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args) {
     std::vector<TParameter> parvec = vsargs_to_parvec(args, nargs);
     return cppyy_object_t(Cppyy::CallConstructor(method, klass, &parvec));
-//    return cppyy_object_t(Cppyy::CallConstructor(method, klass, args));
 }
 
 cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type) {

From pypy.commits at gmail.com  Tue Aug 16 03:34:39 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 16 Aug 2016 00:34:39 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: some changes to pypy abstract for
 pyconza, added progress file for august (py3.5)
Message-ID: <57b2c20f.09afc20a.90904.6eb6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5668:3c35a2b9d370
Date: 2016-08-16 08:47 +0200
http://bitbucket.org/pypy/extradoc/changeset/3c35a2b9d370/

Log:	some changes to pypy abstract for pyconza, added progress file for
	august (py3.5)

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
new file mode 100644
--- /dev/null
+++ b/planning/py3.5/2016-august-progress.rst
@@ -0,0 +1,10 @@
+August 2016
+===========
+
+Planned
+-------
+
+* Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951 (plan_rich)
+
+Finished
+--------
diff --git a/talk/pyconza2016/pypy-abstract.txt b/talk/pyconza2016/pypy-abstract.txt
--- a/talk/pyconza2016/pypy-abstract.txt
+++ b/talk/pyconza2016/pypy-abstract.txt
@@ -3,36 +3,16 @@
 
 In this talk I want to show how you can use PyPy for your benefit.
 It will kick off with a short introduction covering PyPy and its just in time
-compiler. PyPy is the most advanced Python interpreter around (besides CPython)
-XXX
-XXX you seem to say "CPython is more advanced than PyPy" above,
-XXX which doesn't make sense in this sentence because you say
-XXX below that PyPy is much faster than CPython
-XXX
+compiler. PyPy is the most advanced Python interpreter around
 and while it should generally just speed up your programs there is a wide range
 of performance that you can get out of PyPy.
 
 The first part, will cover considerations why one should write Python programs,
 and only spend fractions of the development time to optimize your program.
-XXX
-XXX you mean, you want to explain that developers should write in Python
-XXX and spend only a small part of their time optimizing the program?
-XXX or something else?  if I'm right then you should add below something 
-XXX like "The second part of this session will be about this small part
-XXX of time: in cases where you need it, then I'll show tools that..."
-XXX But I'm not sure that's what you mean because CFFI is not really
-XXX about that: I'm trying to push it as a general solution also for
-XXX CPython, without focusing too much on performance.  Maybe we should
-XXX have this talk be really about PyPy, and then for the other talk
-XXX I should have both CFFI and RevDB?
-XXX
-The second part of this session will show and give you the knowledge and
-tools to inspect and change your program to improve it. We will cover two tools in detail:
-CFFI & VMProf.
-
-Our advanced library CFFI (C Foreign Function Interface) can easily replace
-CPython extension code. VMProf is a platform to inspect you program while it is running,
-imposing very little overhead.
+The second part of this session will be about this small part
+of time: in cases where you need it, then I'll show tools that help you inspect
+and change your program to improve it. We will also dive into one tool more elaboratly.
+VMProf, a platform to inspect your program while it is running, imposing very little overhead.
 
 Throughout the talk real world examples will motivate why PyPy is a viable option
 to optimize you Python programs and present the examples' value to their developers.

From pypy.commits at gmail.com  Tue Aug 16 03:40:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 00:40:59 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Add the formal content of
 milestone 1
Message-ID: <57b2c38b.88711c0a.56af.bed1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5669:bd8ee375d546
Date: 2016-08-16 09:40 +0200
http://bitbucket.org/pypy/extradoc/changeset/bd8ee375d546/

Log:	Add the formal content of milestone 1

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -8,3 +8,41 @@
 
 Finished
 --------
+
+
+
+Milestone 1 (Aug-Sep-Oct 2016)
+------------------------------
+
+We have reached milestone 1 when we have done all the following point,
+possibly minus one of them if it is found during development that
+properly implementing it requires significantly more efforts than
+planned:
+
+* PEP 492, coroutines with async and await syntax.  (The complete PEP
+  is included.)
+
+* PEP 465, a new matrix multiplication operator: a @ b.
+
+* PEP 448, additional unpacking generalizations.
+
+* bytes % args, bytearray % args: PEP 461
+
+* New bytes.hex(), bytearray.hex() and memoryview.hex() methods.
+
+* memoryview now supports tuple indexing
+
+* Generators have a new gi_yieldfrom attribute
+
+* A new RecursionError exception is now raised when maximum recursion
+  depth is reached.
+
+* The new os.scandir() function
+
+* Newly created file descriptors are non-inheritable (PEP 446)
+
+* The marshal format has been made more compact and efficient
+
+* enum: Support for enumeration types (PEP 435).
+
+* pathlib: Object-oriented filesystem paths (PEP 428).

From pypy.commits at gmail.com  Tue Aug 16 04:03:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 01:03:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix site.py
Message-ID: <57b2c8c3.2916c20a.26156.7fb2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86209:982050f81cb9
Date: 2016-08-16 09:11 +0100
http://bitbucket.org/pypy/pypy/changeset/982050f81cb9/

Log:	Fix site.py

diff --git a/lib-python/3/site.py b/lib-python/3/site.py
--- a/lib-python/3/site.py
+++ b/lib-python/3/site.py
@@ -378,8 +378,8 @@
         license = "See https://www.python.org/psf/license/"
         licenseargs = (license, files, dirs)
 
-    builtins.credits = _Printer("credits", credits)
-    builtins.license = _Printer("license", *licenseargs)
+    builtins.credits = _sitebuiltins._Printer("credits", credits)
+    builtins.license = _sitebuiltins._Printer("license", *licenseargs)
 
 def sethelper():
     builtins.help = _sitebuiltins._Helper()

From pypy.commits at gmail.com  Tue Aug 16 04:18:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 01:18:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fixes
Message-ID: <57b2cc5f.c2a5c20a.de387.8190@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86210:f4068c63d5e3
Date: 2016-08-16 09:26 +0100
http://bitbucket.org/pypy/pypy/changeset/f4068c63d5e3/

Log:	fixes

diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py
--- a/lib-python/3/distutils/command/build_ext.py
+++ b/lib-python/3/distutils/command/build_ext.py
@@ -11,7 +11,6 @@
 from distutils.core import Command
 from distutils.errors import *
 from distutils.sysconfig import customize_compiler, get_python_version
-from distutils.sysconfig import get_config_h_filename
 from distutils.dep_util import newer_group
 from distutils.extension import Extension
 from distutils.util import get_platform
@@ -30,6 +29,7 @@
     show_compilers()
 
 def _get_c_extension_suffix():
+    import importlib
     suffixes = importlib.machinery.EXTENSION_SUFFIXES
     return suffixes[0] if suffixes else None
 
@@ -204,6 +204,7 @@
             # this allows distutils on windows to work in the source tree
             if 0:
                 # pypy has no config_h_filename directory
+                from distutils.sysconfig import get_config_h_filename
                 self.include_dirs.append(os.path.dirname(get_config_h_filename()))
             _sys_home = getattr(sys, '_home', None)
             if _sys_home:

From pypy.commits at gmail.com  Tue Aug 16 04:25:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 01:25:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Revert all changes to this from Python
 3.5.1. This mainly reverts
Message-ID: <57b2ce16.a427c20a.44913.8825@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86211:4c4ec4987626
Date: 2016-08-16 10:25 +0200
http://bitbucket.org/pypy/pypy/changeset/4c4ec4987626/

Log:	Revert all changes to this from Python 3.5.1. This mainly reverts
	1341a432e134, which I think is not needed any more in 3.5

diff --git a/lib-python/3/code.py b/lib-python/3/code.py
--- a/lib-python/3/code.py
+++ b/lib-python/3/code.py
@@ -140,32 +140,15 @@
         sys.last_type, sys.last_value, last_tb = ei = sys.exc_info()
         sys.last_traceback = last_tb
         try:
-            lines = []
-            for value, tb in traceback._iter_chain(*ei[1:]):
-                if isinstance(value, str):
-                    lines.append(value)
-                    lines.append('\n')
-                    continue
-                if tb:
-                    tblist = traceback.extract_tb(tb)
-                    if tb is last_tb:
-                        # The last traceback includes the frame we
-                        # exec'd in
-                        del tblist[:1]
-                    tblines = traceback.format_list(tblist)
-                    if tblines:
-                        lines.append("Traceback (most recent call last):\n")
-                        lines.extend(tblines)
-                lines.extend(traceback.format_exception_only(type(value),
-                                                             value))
+            lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next)
+            if sys.excepthook is sys.__excepthook__:
+                self.write(''.join(lines))
+            else:
+                # If someone has set sys.excepthook, we let that take precedence
+                # over self.write
+                sys.excepthook(ei[0], ei[1], last_tb)
         finally:
-            tblist = last_tb = ei = None
-        if sys.excepthook is sys.__excepthook__:
-            self.write(''.join(lines))
-        else:
-            # If someone has set sys.excepthook, we let that take precedence
-            # over self.write
-            sys.excepthook(sys.last_type, sys.last_value, last_tb)
+            last_tb = ei = None
 
     def write(self, data):
         """Write a string.

From pypy.commits at gmail.com  Tue Aug 16 04:30:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 01:30:23 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Update
Message-ID: <57b2cf1f.54bc1c0a.2469c.c880@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5670:1958a42c9de1
Date: 2016-08-16 10:30 +0200
http://bitbucket.org/pypy/extradoc/changeset/1958a42c9de1/

Log:	Update

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -6,6 +6,10 @@
 
 * Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951 (plan_rich)
 
+* Make a translated py3.5 actually work a bit (currently we get
+  systematic failures), up to the point where we can meaningfully
+  run the lib-python tests (arigo)
+
 Finished
 --------
 

From pypy.commits at gmail.com  Tue Aug 16 05:14:12 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 16 Aug 2016 02:14:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: hex method for memoryview +
 tests
Message-ID: <57b2d964.45d11c0a.31711.d5e6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86212:8e69fe818876
Date: 2016-08-16 11:13 +0200
http://bitbucket.org/pypy/pypy/changeset/8e69fe818876/

Log:	hex method for memoryview + tests

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -491,6 +491,24 @@
         i += 2
     return data
 
+HEXDIGITS = "0123456789abcdef"
+PY_SIZE_T_MAX = 2**(rffi.sizeof(rffi.SIZE_T)*8)-1
+
+def _array_to_hexstring(space, buf):
+    length = buf.getlength()
+    hexstring = StringBuilder(length*2)
+
+    if length > PY_SIZE_T_MAX/2:
+        raise OperationError(space.w_MemoryError)
+
+    for i in range(length):
+        byte = ord(buf.getitem(i))
+        c = (byte >> 4 & 0xf)
+        hexstring.append(HEXDIGITS[c])
+        c = (byte & 0xf)
+        hexstring.append(HEXDIGITS[c])
+
+    return space.wrap(hexstring.build())
 
 class BytearrayDocstrings:
     """bytearray(iterable_of_ints) -> bytearray
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -232,6 +232,11 @@
         newitemsize = self.get_native_fmtchar(fmt)
         return W_MemoryView(self.buf, fmt, newitemsize)
 
+    def descr_hex(self, space):
+        from pypy.objspace.std.bytearrayobject import _array_to_hexstring
+        self._check_released(space)
+        return _array_to_hexstring(space, self.buf)
+
 
 W_MemoryView.typedef = TypeDef(
     "memoryview",
@@ -250,6 +255,7 @@
     __exit__    = interp2app(W_MemoryView.descr_exit),
     __weakref__ = make_weakref_descr(W_MemoryView),
     cast        = interp2app(W_MemoryView.descr_cast),
+    hex         = interp2app(W_MemoryView.descr_hex),
     tobytes     = interp2app(W_MemoryView.descr_tobytes),
     tolist      = interp2app(W_MemoryView.descr_tolist),
     release     = interp2app(W_MemoryView.descr_release),
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -18,6 +18,7 @@
         assert len(w) == 2
         exc = raises(NotImplementedError, "v[0:2:2]")
         assert str(exc.value) == ""
+        exc = raises(TypeError, "memoryview('foobar')")
 
     def test_rw(self):
         data = bytearray(b'abcefg')
@@ -161,3 +162,6 @@
         raises(ValueError, memoryview(b"foobar")._pypy_raw_address)
         a = memoryview(bytearray(b"foobar"))._pypy_raw_address()
         assert a != 0
+
+    def test_hex(self):
+        assert memoryview(b"abc").hex() == u'616263'

From pypy.commits at gmail.com  Tue Aug 16 05:15:52 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 16 Aug 2016 02:15:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Merge with py3.5
Message-ID: <57b2d9c8.45c8c20a.dc13a.9a76@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86213:a955efeebd40
Date: 2016-08-16 11:14 +0200
http://bitbucket.org/pypy/pypy/changeset/a955efeebd40/

Log:	Merge with py3.5

diff too long, truncating to 2000 out of 4303 lines

diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib-python/3/code.py b/lib-python/3/code.py
--- a/lib-python/3/code.py
+++ b/lib-python/3/code.py
@@ -140,32 +140,15 @@
         sys.last_type, sys.last_value, last_tb = ei = sys.exc_info()
         sys.last_traceback = last_tb
         try:
-            lines = []
-            for value, tb in traceback._iter_chain(*ei[1:]):
-                if isinstance(value, str):
-                    lines.append(value)
-                    lines.append('\n')
-                    continue
-                if tb:
-                    tblist = traceback.extract_tb(tb)
-                    if tb is last_tb:
-                        # The last traceback includes the frame we
-                        # exec'd in
-                        del tblist[:1]
-                    tblines = traceback.format_list(tblist)
-                    if tblines:
-                        lines.append("Traceback (most recent call last):\n")
-                        lines.extend(tblines)
-                lines.extend(traceback.format_exception_only(type(value),
-                                                             value))
+            lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next)
+            if sys.excepthook is sys.__excepthook__:
+                self.write(''.join(lines))
+            else:
+                # If someone has set sys.excepthook, we let that take precedence
+                # over self.write
+                sys.excepthook(ei[0], ei[1], last_tb)
         finally:
-            tblist = last_tb = ei = None
-        if sys.excepthook is sys.__excepthook__:
-            self.write(''.join(lines))
-        else:
-            # If someone has set sys.excepthook, we let that take precedence
-            # over self.write
-            sys.excepthook(sys.last_type, sys.last_value, last_tb)
+            last_tb = ei = None
 
     def write(self, data):
         """Write a string.
diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py
--- a/lib-python/3/distutils/command/build_ext.py
+++ b/lib-python/3/distutils/command/build_ext.py
@@ -11,7 +11,6 @@
 from distutils.core import Command
 from distutils.errors import *
 from distutils.sysconfig import customize_compiler, get_python_version
-from distutils.sysconfig import get_config_h_filename
 from distutils.dep_util import newer_group
 from distutils.extension import Extension
 from distutils.util import get_platform
@@ -30,6 +29,7 @@
     show_compilers()
 
 def _get_c_extension_suffix():
+    import importlib
     suffixes = importlib.machinery.EXTENSION_SUFFIXES
     return suffixes[0] if suffixes else None
 
@@ -204,6 +204,7 @@
             # this allows distutils on windows to work in the source tree
             if 0:
                 # pypy has no config_h_filename directory
+                from distutils.sysconfig import get_config_h_filename
                 self.include_dirs.append(os.path.dirname(get_config_h_filename()))
             _sys_home = getattr(sys, '_home', None)
             if _sys_home:
diff --git a/lib-python/3/site.py b/lib-python/3/site.py
--- a/lib-python/3/site.py
+++ b/lib-python/3/site.py
@@ -378,8 +378,8 @@
         license = "See https://www.python.org/psf/license/"
         licenseargs = (license, files, dirs)
 
-    builtins.credits = _Printer("credits", credits)
-    builtins.license = _Printer("license", *licenseargs)
+    builtins.credits = _sitebuiltins._Printer("credits", credits)
+    builtins.license = _sitebuiltins._Printer("license", *licenseargs)
 
 def sethelper():
     builtins.help = _sitebuiltins._Helper()
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -105,3 +105,26 @@
 .. branch: ep2016sprint
 
 Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -114,8 +114,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1322,6 +1322,25 @@
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
 
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts
+
     def test_call_function_var(self):
         source = """call(*me)"""
         code, blocks = generate_function_code(source, self.space)
diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py
--- a/pypy/interpreter/astcompiler/test/test_validate.py
+++ b/pypy/interpreter/astcompiler/test/test_validate.py
@@ -38,9 +38,8 @@
         self.mod(m, "must have Load context", "eval")
 
     def _check_arguments(self, fac, check):
-        def arguments(args=None, vararg=None, varargannotation=None,
-                      kwonlyargs=None, kwarg=None, kwargannotation=None,
-                      defaults=None, kw_defaults=None):
+        def arguments(args=None, vararg=None, kwonlyargs=None,
+                      kw_defaults=None, kwarg=None, defaults=None):
             if args is None:
                 args = []
             if kwonlyargs is None:
@@ -49,20 +48,12 @@
                 defaults = []
             if kw_defaults is None:
                 kw_defaults = []
-            args = ast.arguments(args, vararg, varargannotation, kwonlyargs,
-                                 kwarg, kwargannotation, defaults, kw_defaults)
+            args = ast.arguments(args, vararg, kwonlyargs,
+                                 kw_defaults, kwarg, defaults)
             return fac(args)
         args = [ast.arg("x", ast.Name("x", ast.Store, 0, 0))]
         check(arguments(args=args), "must have Load context")
-        check(arguments(varargannotation=ast.Num(self.space.wrap(3), 0, 0)),
-              "varargannotation but no vararg")
-        check(arguments(varargannotation=ast.Name("x", ast.Store, 0, 0), vararg="x"),
-                         "must have Load context")
         check(arguments(kwonlyargs=args), "must have Load context")
-        check(arguments(kwargannotation=ast.Num(self.space.wrap(42), 0, 0)),
-                       "kwargannotation but no kwarg")
-        check(arguments(kwargannotation=ast.Name("x", ast.Store, 0, 0),
-                          kwarg="x"), "must have Load context")
         check(arguments(defaults=[ast.Num(self.space.wrap(3), 0, 0)]),
                        "more positional defaults than args")
         check(arguments(kw_defaults=[ast.Num(self.space.wrap(4), 0, 0)]),
@@ -77,7 +68,7 @@
                           "must have Load context")
 
     def test_funcdef(self):
-        a = ast.arguments([], None, None, [], None, None, [], [])
+        a = ast.arguments([], None, [], [], None, [])
         f = ast.FunctionDef("x", a, [], [], None, 0, 0)
         self.stmt(f, "empty body on FunctionDef")
         f = ast.FunctionDef("x", a, [ast.Pass(0, 0)], [ast.Name("x", ast.Store, 0, 0)],
@@ -91,8 +82,7 @@
         self._check_arguments(fac, self.stmt)
 
     def test_classdef(self):
-        def cls(bases=None, keywords=None, starargs=None, kwargs=None,
-                body=None, decorator_list=None):
+        def cls(bases=None, keywords=None, body=None, decorator_list=None):
             if bases is None:
                 bases = []
             if keywords is None:
@@ -101,16 +91,12 @@
                 body = [ast.Pass(0, 0)]
             if decorator_list is None:
                 decorator_list = []
-            return ast.ClassDef("myclass", bases, keywords, starargs,
-                                kwargs, body, decorator_list, 0, 0)
+            return ast.ClassDef("myclass", bases, keywords,
+                                body, decorator_list, 0, 0)
         self.stmt(cls(bases=[ast.Name("x", ast.Store, 0, 0)]),
                   "must have Load context")
         self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store, 0, 0))]),
                   "must have Load context")
-        self.stmt(cls(starargs=ast.Name("x", ast.Store, 0, 0)),
-                  "must have Load context")
-        self.stmt(cls(kwargs=ast.Name("x", ast.Store, 0, 0)),
-                  "must have Load context")
         self.stmt(cls(body=[]), "empty body on ClassDef")
         self.stmt(cls(body=[None]), "None disallowed")
         self.stmt(cls(decorator_list=[ast.Name("x", ast.Store, 0, 0)]),
@@ -250,7 +236,7 @@
         self.expr(u, "must have Load context")
 
     def test_lambda(self):
-        a = ast.arguments([], None, None, [], None, None, [], [])
+        a = ast.arguments([], None, [], [], None, [])
         self.expr(ast.Lambda(a, ast.Name("x", ast.Store, 0, 0), 0, 0),
                   "must have Load context")
         def fac(args):
@@ -343,20 +329,12 @@
         func = ast.Name("x", ast.Load, 0, 0)
         args = [ast.Name("y", ast.Load, 0, 0)]
         keywords = [ast.keyword("w", ast.Name("z", ast.Load, 0, 0))]
-        stararg = ast.Name("p", ast.Load, 0, 0)
-        kwarg = ast.Name("q", ast.Load, 0, 0)
-        call = ast.Call(ast.Name("x", ast.Store, 0, 0), args, keywords, stararg,
-                        kwarg, 0, 0)
+        call = ast.Call(ast.Name("x", ast.Store, 0, 0), args, keywords, 0, 0)
         self.expr(call, "must have Load context")
-        call = ast.Call(func, [None], keywords, stararg, kwarg, 0, 0)
+        call = ast.Call(func, [None], keywords, 0, 0)
         self.expr(call, "None disallowed")
         bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store, 0, 0))]
-        call = ast.Call(func, args, bad_keywords, stararg, kwarg, 0, 0)
-        self.expr(call, "must have Load context")
-        call = ast.Call(func, args, keywords, ast.Name("z", ast.Store, 0, 0), kwarg, 0, 0)
-        self.expr(call, "must have Load context")
-        call = ast.Call(func, args, keywords, stararg,
-                        ast.Name("w", ast.Store, 0, 0), 0, 0)
+        call = ast.Call(func, args, bad_keywords, 0, 0)
         self.expr(call, "must have Load context")
 
     def test_num(self):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1375,10 +1375,11 @@
         for i in range(itemcount, 0, -1):
             w_item = self.peekvalue(i-1)
             # cannot use w_sum.update, w_item might not be a set
-            iterator = w_item.itervalues()
+            iterator = space.iter(w_item)
             while True:
-                w_value = iterator.next_value()
-                if w_value is None:
+                try:
+                    w_value = space.next(iterator)
+                except OperationError:
                     break
                 w_sum.add(w_value)
         while itemcount != 0:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -167,6 +167,12 @@
         py.test.raises(SyntaxError, self.parse, 'f()\n# blah\nblah()', "single")
         py.test.raises(SyntaxError, self.parse, 'f()\nxy # blah\nblah()', "single")
         py.test.raises(SyntaxError, self.parse, 'x = 5 # comment\nx = 6\n', "single")
+    
+    def test_unpack(self):
+        self.parse('[*{2}, 3, *[4]]')
+        self.parse('{*{2}, 3, *[4]}')
+        self.parse('{**{}, 3:4, **{5:6, 7:8}}')
+        self.parse('f(2, *a, *b, **b, **c, **d)')
 
     def test_async_await(self):
         self.parse("async def coro(): await func")
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -209,7 +209,6 @@
         self.check(['-c', 'pass'], {'PYTHONNOUSERSITE': '1'}, sys_argv=['-c'],
                    run_command='pass', **expected)
 
-
 class TestInteraction:
     """
     These tests require pexpect (UNIX-only).
@@ -1152,4 +1151,3 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -134,7 +134,7 @@
         assert self.space.lookup(w_instance, "gobbledygook") is None
         w_instance = self.space.appexec([], """():
             class Lookup(object):
-                "bla" 
+                "bla"
             return Lookup()""")
         assert self.space.str_w(self.space.lookup(w_instance, "__doc__")) == "bla"
 
@@ -148,7 +148,7 @@
         assert is_callable(w_func)
         w_lambda_func = self.space.appexec([], "(): return lambda: True")
         assert is_callable(w_lambda_func)
-        
+
         w_instance = self.space.appexec([], """():
             class Call(object):
                 def __call__(self): pass
@@ -308,7 +308,7 @@
 
     def test_call_obj_args(self):
         from pypy.interpreter.argument import Arguments
-        
+
         space = self.space
 
         w_f = space.appexec([], """():
@@ -333,7 +333,7 @@
         assert w_x is w_9
         assert w_y is w_1
 
-        w_res = space.call_obj_args(w_a, w_9, Arguments(space, []))        
+        w_res = space.call_obj_args(w_a, w_9, Arguments(space, []))
         assert w_res is w_9
 
     def test_compare_by_iteration(self):
@@ -383,7 +383,7 @@
         assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('g')))
         assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('h')))
 
-class TestModuleMinimal: 
+class TestModuleMinimal:
     def test_sys_exists(self):
         assert self.space.sys
 
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -18,3 +18,31 @@
         loop.run_until_complete(f())
         print("done with async loop")
         """
+    
+    def test_asynchronous_context_managers(self):
+        """
+import encodings.idna
+import asyncio
+
+class Corotest(object):
+    def __init__(self):
+        self.res = "-"
+    
+    async def coro(self, name, lock):
+        self.res += ' coro {}: waiting for lock -'.format(name)
+        async with lock:
+            self.res += ' coro {}: holding the lock -'.format(name)
+            await asyncio.sleep(1)
+            self.res += ' coro {}: releasing the lock -'.format(name)
+
+cor = Corotest()
+loop = asyncio.get_event_loop()
+lock = asyncio.Lock()
+coros = asyncio.gather(cor.coro(1, lock), cor.coro(2, lock))
+try:
+    loop.run_until_complete(coros)
+finally:
+    loop.close()
+
+assert cor.res == "- coro 1: waiting for lock - coro 1: holding the lock - coro 2: waiting for lock - coro 1: releasing the lock - coro 2: holding the lock - coro 2: releasing the lock -"
+        """
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.7.0"
+VERSION = "1.8.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -157,11 +157,13 @@
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
         try:
+            keepalives = [None] * len(args_w)    # None or strings
             for i in range(len(args_w)):
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                 w_obj = args_w[i]
                 argtype = self.fargs[i]
-                if argtype.convert_argument_from_object(data, w_obj):
+                if argtype.convert_argument_from_object(data, w_obj,
+                                                        keepalives, i):
                     # argtype is a pointer type, and w_obj a list/tuple/str
                     mustfree_max_plus_1 = i + 1
 
@@ -177,9 +179,13 @@
                 if isinstance(argtype, W_CTypePointer):
                     data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                     flag = get_mustfree_flag(data)
+                    raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                     if flag == 1:
-                        raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
+                    elif flag >= 4:
+                        value = keepalives[i]
+                        assert value is not None
+                        rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
             lltype.free(buffer, flavor='raw')
             keepalive_until_here(args_w)
         return w_res
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -83,7 +83,7 @@
         raise oefmt(space.w_TypeError, "cannot initialize cdata '%s'",
                     self.name)
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         self.convert_from_object(cdata, w_ob)
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -16,8 +16,8 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
+    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
     length = -1
 
     def __init__(self, space, size, extra, extra_position, ctitem,
@@ -30,6 +30,9 @@
         #  - for functions, it is the return type
         self.ctitem = ctitem
         self.can_cast_anything = could_cast_anything and ctitem.cast_anything
+        self.accept_str = (self.can_cast_anything or
+                            (ctitem.is_primitive_integer and
+                             ctitem.size == rffi.sizeof(lltype.Char)))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -72,9 +75,7 @@
                 pass
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
-        elif (self.can_cast_anything or
-              (self.ctitem.is_primitive_integer and
-               self.ctitem.size == rffi.sizeof(lltype.Char))):
+        elif self.accept_str:
             if not space.isinstance_w(w_ob, space.w_str):
                 raise self._convert_error("bytes or list or tuple", w_ob)
             s = space.str_w(w_ob)
@@ -262,8 +263,16 @@
         else:
             return lltype.nullptr(rffi.CCHARP.TO)
 
-    def _prepare_pointer_call_argument(self, w_init, cdata):
+    def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
         space = self.space
+        if self.accept_str and space.isinstance_w(w_init, space.w_str):
+            # special case to optimize strings passed to a "char *" argument
+            value = space.bytes_w(w_init)
+            keepalives[i] = value
+            buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
+            rffi.cast(rffi.CCHARPP, cdata)[0] = buf
+            return ord(buf_flag)    # 4, 5 or 6
+        #
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
@@ -300,10 +309,11 @@
         rffi.cast(rffi.CCHARPP, cdata)[0] = result
         return 1
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
         result = (not isinstance(w_ob, cdataobj.W_CData) and
-                  self._prepare_pointer_call_argument(w_ob, cdata))
+                  self._prepare_pointer_call_argument(w_ob, cdata,
+                                                      keepalives, i))
         if result == 0:
             self.convert_from_object(cdata, w_ob)
         set_mustfree_flag(cdata, result)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -353,7 +353,7 @@
 'array.array' or numpy arrays."""
         #
         w_ctchara = newtype._new_chara_type(self.space)
-        return func.from_buffer(self.space, w_ctchara, w_python_buffer)
+        return func._from_buffer(self.space, w_ctchara, w_python_buffer)
 
 
     @unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated
+from rpython.rlib import jit
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -132,17 +133,66 @@
         raise oefmt(space.w_TypeError,
                     "needs 'char[]', got '%s'", w_ctype.name)
     #
+    return _from_buffer(space, w_ctype, w_x)
+
+def _from_buffer(space, w_ctype, w_x):
     buf = _fetch_as_read_buffer(space, w_x)
-    try:
-        _cdata = buf.get_raw_address()
-    except ValueError:
-        raise oefmt(space.w_TypeError,
-                    "from_buffer() got a '%T' object, which supports the "
-                    "buffer interface but cannot be rendered as a plain "
-                    "raw address on PyPy", w_x)
+    if space.isinstance_w(w_x, space.w_str):
+        _cdata = get_raw_address_of_string(space, w_x)
+    else:
+        try:
+            _cdata = buf.get_raw_address()
+        except ValueError:
+            raise oefmt(space.w_TypeError,
+                        "from_buffer() got a '%T' object, which supports the "
+                        "buffer interface but cannot be rendered as a plain "
+                        "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
 
+# ____________________________________________________________
+
+class RawBytes(object):
+    def __init__(self, string):
+        self.ptr = rffi.str2charp(string, track_allocation=False)
+    def __del__(self):
+        rffi.free_charp(self.ptr, track_allocation=False)
+
+class RawBytesCache(object):
+    def __init__(self, space):
+        from pypy.interpreter.baseobjspace import W_Root
+        from rpython.rlib import rweakref
+        self.wdict = rweakref.RWeakKeyDictionary(W_Root, RawBytes)
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(space, w_x):
+    """Special case for ffi.from_buffer(string).  Returns a 'char *' that
+    is valid as long as the string object is alive.  Two calls to
+    ffi.from_buffer(same_string) are guaranteed to return the same pointer.
+    """
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rtyper.lltypesystem.rstr import STR
+    from rpython.rtyper.lltypesystem import llmemory
+    from rpython.rlib import rgc
+
+    cache = space.fromcache(RawBytesCache)
+    rawbytes = cache.wdict.get(w_x)
+    if rawbytes is None:
+        data = space.str_w(w_x)
+        if we_are_translated() and not rgc.can_move(data):
+            lldata = llstr(data)
+            data_start = (llmemory.cast_ptr_to_adr(lldata) +
+                          rffi.offsetof(STR, 'chars') +
+                          llmemory.itemoffsetof(STR.chars, 0))
+            data_start = rffi.cast(rffi.CCHARP, data_start)
+            data_start[len(data)] = '\x00'   # write the final extra null
+            return data_start
+        rawbytes = RawBytes(data)
+        cache.wdict.set(w_x, rawbytes)
+    return rawbytes.ptr
+
+# ____________________________________________________________
+
 
 def unsafe_escaping_ptr_for_ptr_or_array(w_cdata):
     if not w_cdata.ctype.is_nonfunc_pointer_or_array:
diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py
--- a/pypy/module/_cffi_backend/parse_c_type.py
+++ b/pypy/module/_cffi_backend/parse_c_type.py
@@ -97,11 +97,8 @@
                                   [rffi.INT], rffi.CCHARP)
 
 def parse_c_type(info, input):
-    p_input = rffi.str2charp(input)
-    try:
+    with rffi.scoped_view_charp(input) as p_input:
         res = ll_parse_c_type(info, p_input)
-    finally:
-        rffi.free_charp(p_input)
     return rffi.cast(lltype.Signed, res)
 
 NULL_CTX = lltype.nullptr(PCTX.TO)
@@ -130,15 +127,13 @@
     return rffi.getintfield(src_ctx, 'c_num_types')
 
 def search_in_globals(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_globals(ctx, c_name,
-                                  rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_globals(ctx, c_name,
+                                      rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
 
 def search_in_struct_unions(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_struct_unions(ctx, c_name,
-                                        rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_struct_unions(ctx, c_name,
+                                            rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3330,13 +3330,18 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharA = new_array_type(BCharP, None)
-    py.test.raises(TypeError, from_buffer, BCharA, b"foo")
+    p1 = from_buffer(BCharA, b"foo")
+    assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # from_buffer(buffer(b"foo")) does not work, because it's not
+        # implemented on pypy; only from_buffer(b"foo") works.
         py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
         py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
     try:
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -402,21 +402,20 @@
             _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
         from rpython.rlib import rwin32
 
-        charp = rffi.str2charp(buf)
-        written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                    flavor='raw')
-        try:
-            result = _WriteFile(
-                self.handle, rffi.ptradd(charp, offset),
-                size, written_ptr, rffi.NULL)
+        with rffi.scoped_view_charp(buf) as charp:
+            written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
+                                        flavor='raw')
+            try:
+                result = _WriteFile(
+                    self.handle, rffi.ptradd(charp, offset),
+                    size, written_ptr, rffi.NULL)
 
-            if (result == 0 and
-                rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
-                raise oefmt(space.w_ValueError,
-                            "Cannot send %d bytes over connection", size)
-        finally:
-            rffi.free_charp(charp)
-            lltype.free(written_ptr, flavor='raw')
+                if (result == 0 and
+                    rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
+                    raise oefmt(space.w_ValueError,
+                                "Cannot send %d bytes over connection", size)
+            finally:
+                lltype.free(written_ptr, flavor='raw')
 
     def do_recv_string(self, space, buflength, maxlength):
         from pypy.module._multiprocessing.interp_win32 import (
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -15,8 +15,9 @@
 
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
-        includes=['unistd.h', 'sys/syscall.h'])
+        includes=['unistd.h', 'sys/syscall.h', 'sys/stat.h'])
     HAVE_SYS_SYSCALL_H = platform.Has("syscall")
+    HAVE_SYS_STAT_H = platform.Has("stat")
     HAVE_SETSID = platform.Has("setsid")
 
 config = platform.configure(CConfig)
@@ -29,6 +30,8 @@
 compile_extra = []
 if config['HAVE_SYS_SYSCALL_H']:
     compile_extra.append("-DHAVE_SYS_SYSCALL_H")
+if config['HAVE_SYS_STAT_H']:
+    compile_extra.append("-DHAVE_SYS_STAT_H")
 if config['HAVE_SETSID']:
     compile_extra.append("-DHAVE_SETSID")
 
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,6 +1,7 @@
 import sys, os
-import py
+import pytest
 from pypy.tool.pytest.objspace import gettestobjspace
+from pypy.interpreter.gateway import interp2app
 from rpython.tool.udir import udir
 from rpython.rlib import rsocket
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -13,8 +14,6 @@
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
     mod.path = udir.join('fd')
     mod.path.write('fo')
-    mod.raises = py.test.raises # make raises available from app-level tests
-    mod.skip = py.test.skip
 
 def test_gethostname():
     host = space.appexec([w_socket], "(_socket): return _socket.gethostname()")
@@ -42,7 +41,7 @@
     for host in ["localhost", "127.0.0.1", "::1"]:
         if host == "::1" and not ipv6:
             from pypy.interpreter.error import OperationError
-            with py.test.raises(OperationError):
+            with pytest.raises(OperationError):
                 space.appexec([w_socket, space.wrap(host)],
                               "(_socket, host): return _socket.gethostbyaddr(host)")
             continue
@@ -58,14 +57,14 @@
     assert space.unwrap(port) == 25
     # 1 arg version
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyname second argument is not optional before python 2.4")
+        pytest.skip("getservbyname second argument is not optional before python 2.4")
     port = space.appexec([w_socket, space.wrap(name)],
                         "(_socket, name): return _socket.getservbyname(name)")
     assert space.unwrap(port) == 25
 
 def test_getservbyport():
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyport does not exist before python 2.4")
+        pytest.skip("getservbyport does not exist before python 2.4")
     port = 25
     # 2 args version
     name = space.appexec([w_socket, space.wrap(port)],
@@ -139,7 +138,7 @@
 
 def test_pton_ntop_ipv4():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     tests = [
         ("123.45.67.89", "\x7b\x2d\x43\x59"),
         ("0.0.0.0", "\x00" * 4),
@@ -155,9 +154,9 @@
 
 def test_ntop_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -176,9 +175,9 @@
 
 def test_pton_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -197,7 +196,7 @@
         assert space.unwrap(w_packed) == packed
 
 def test_has_ipv6():
-    py.test.skip("has_ipv6 is always True on PyPy for now")
+    pytest.skip("has_ipv6 is always True on PyPy for now")
     res = space.appexec([w_socket], "(_socket): return _socket.has_ipv6")
     assert space.unwrap(res) == socket.has_ipv6
 
@@ -231,7 +230,7 @@
 def test_addr_raw_packet():
     from pypy.module._socket.interp_socket import addr_as_object
     if not hasattr(rsocket._c, 'sockaddr_ll'):
-        py.test.skip("posix specific test")
+        pytest.skip("posix specific test")
     # HACK: To get the correct interface number of lo, which in most cases is 1,
     # but can be anything (i.e. 39), we need to call the libc function
     # if_nametoindex to get the correct index
@@ -653,11 +652,11 @@
 class AppTestNetlink:
     def setup_class(cls):
         if not hasattr(os, 'getpid'):
-            py.test.skip("AF_NETLINK needs os.getpid()")
+            pytest.skip("AF_NETLINK needs os.getpid()")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_NETLINK')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_NETLINK on this platform")
+            pytest.skip("no AF_NETLINK on this platform")
         cls.space = space
 
     def test_connect_to_kernel_netlink_routing_socket(self):
@@ -673,11 +672,11 @@
 class AppTestPacket:
     def setup_class(cls):
         if not hasattr(os, 'getuid') or os.getuid() != 0:
-            py.test.skip("AF_PACKET needs to be root for testing")
+            pytest.skip("AF_PACKET needs to be root for testing")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_PACKET')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_PACKET on this platform")
+            pytest.skip("no AF_PACKET on this platform")
         cls.space = space
 
     def test_convert_between_tuple_and_sockaddr_ll(self):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -146,7 +146,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
@@ -158,7 +158,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
@@ -192,7 +192,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
@@ -204,7 +204,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
@@ -239,7 +239,7 @@
 
         Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
         bound on the entropy contained in string."""
-        with rffi.scoped_str2charp(string) as buf:
+        with rffi.scoped_nonmovingbuffer(string) as buf:
             libssl_RAND_add(buf, len(string), entropy)
 
     def _RAND_bytes(space, n, pseudo):
diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -537,9 +537,8 @@
     releasegil=ts_helper,
     compilation_info=backend.eci)
 def c_charp2stdstring(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2stdstring(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2stdstring(charp)
     return result
 _c_stdstring2stdstring = rffi.llexternal(
     "cppyy_stdstring2stdstring",
diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -82,9 +82,8 @@
     releasegil=ts_helper,
     compilation_info=eci)
 def c_charp2TString(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2TString(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2TString(charp)
     return result
 _c_TString2TString = rffi.llexternal(
     "cppyy_TString2TString",
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -65,6 +65,7 @@
                 else:    # only other use is sring
                     n = len(obj._string)
                     assert raw_string == rffi.cast(rffi.CCHARP, 0)
+                    # XXX could use rffi.get_nonmovingbuffer_final_null()
                     raw_string = rffi.str2charp(obj._string)
                     data = rffi.cast(rffi.CCHARPP, data)
                     data[0] = raw_string
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,6 +11,9 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+        c_fdopen, c_fileno,
+        c_fopen)# for tests
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import NameManager
@@ -84,44 +87,32 @@
 assert CONST_WSTRING == rffi.CWCHARP
 
 # FILE* interface
-FILEP = rffi.COpaquePtr('FILE')
 
 if sys.platform == 'win32':
     dash = '_'
 else:
     dash = ''
-fileno = rffi.llexternal(dash + 'fileno', [FILEP], rffi.INT)
-fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
-fdopen = rffi.llexternal(dash + 'fdopen', [rffi.INT, CONST_STRING],
-                  FILEP, save_err=rffi.RFFI_SAVE_ERRNO)
 
-_fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
-    if not is_valid_fd(fileno(fp)):
+    if not is_valid_fd(c_fileno(fp)):
         return -1
-    return _fclose(fp)
+    return c_fclose(fp)
 
-_fwrite = rffi.llexternal('fwrite',
-                         [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                         rffi.SIZE_T)
 def fwrite(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fwrite(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fwrite(buf, sz, n, fp)
 
-_fread = rffi.llexternal('fread',
-                        [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                        rffi.SIZE_T)
 def fread(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fread(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fread(buf, sz, n, fp)
 
 _feof = rffi.llexternal('feof', [FILEP], rffi.INT)
 def feof(fp):
-    validate_fd(fileno(fp))
+    validate_fd(c_fileno(fp))
     return _feof(fp)
 
 def is_valid_fp(fp):
-    return is_valid_fd(fileno(fp))
+    return is_valid_fd(c_fileno(fp))
 
 pypy_decl = 'pypy_decl.h'
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -96,7 +96,8 @@
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
             py_str.c_ob_size, len(s))
-    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    with rffi.scoped_nonmovingbuffer(s) as s_ptr:
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
     py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
diff --git a/pypy/module/cpyext/c-api.txt b/pypy/module/cpyext/c-api.txt
deleted file mode 100644
--- a/pypy/module/cpyext/c-api.txt
+++ /dev/null
@@ -1,71 +0,0 @@
-Reference Count
-===============
-
-XXX
-
-Borrowed References
-===================
-
-XXX
-
-PyStringObject support
-======================
-
-The problem
------------
-
-PyString_AsString() returns a (non-movable) pointer to the underlying
-buffer, whereas pypy strings are movable.  C code may temporarily
-store this address and use it, as long as it owns a reference to the
-PyObject.  There is no "release" function to specify that the pointer
-is not needed any more.
-
-Note that the pointer may be used to fill the initial value of
-string. This is valid only when the string was just allocated, and is
-not used elsewhere.
-
-Proposed solution
------------------
-
-Our emulation of the PyStringObject contains an additional member: a
-pointer to a char buffer; it may be NULL.
-
-- A string allocated by pypy will be converted into a PyStringObject
-  with a NULL buffer.  When PyString_AsString() is called, memory is
-  allocated (with flavor='raw') and content is copied.
-
-- A string allocated with PyString_FromStringAndSize(NULL, size) will
-  allocate a buffer with the specified size, but the reference won't
-  be stored in the global map py_objects_r2w; there won't be a
-  corresponding object in pypy.  When from_ref() or Py_INCREF() is
-  called, the pypy string is created, and added in py_objects_r2w.
-  The buffer is then supposed to be immutable.
-
-- _PyString_Resize works only on not-yet-pypy'd strings, and returns a
-  similar object.
-
-- PyString_Size don't need to force the object. (in this case, another
-  "size" member is needed)
-
-- There could be an (expensive!) check in from_ref() that the buffer
-  still corresponds to the pypy gc-managed string.
-
-PySequence_Fast support
-======================
-There are five functions for fast sequence access offered by the CPython API:
-
-PyObject* PySequence_Fast(PyObject *o, const char *m)
-
-PyObject* PySequence_Fast_GET_ITEM(	PyObject *o, int i)
-
-PyObject** PySequence_Fast_ITEMS(	PyObject *o)
-
-PyObject* PySequence_ITEM(	PyObject *o, int i)
-
-int PySequence_Fast_GET_SIZE(	PyObject *o)
-
-PyPy supports four of these, but does not support PySequence_Fast_ITEMS.
-(Various ways to support PySequence_Fast_ITEMS were considered. They all had
-two things in common: they would have taken a lot of work, and they would have
-resulted in incomplete semantics or in poor performance. We decided that a slow
-implementation of PySequence_Fast_ITEMS was not very useful.)
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -25,6 +25,8 @@
                          flavor='raw',
                          add_memory_pressure=True)
 
+realloc = rffi.llexternal('realloc', [rffi.VOIDP, rffi.SIZE_T], rffi.VOIDP)
+
 @cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
@@ -32,7 +34,7 @@
                          flavor='raw',
                          add_memory_pressure=True)
     # XXX FIXME
-    return lltype.nullptr(rffi.VOIDP.TO)
+    return realloc(ptr, size)
 
 @cpython_api([rffi.VOIDP], lltype.Void)
 def PyObject_Free(space, ptr):
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -10,7 +10,7 @@
 from pypy.objspace.std import tupleobject
 
 from pypy.module.cpyext.tupleobject import PyTuple_Check, PyTuple_SetItem
-from pypy.module.cpyext.object import Py_IncRef, Py_DecRef
+from pypy.module.cpyext.pyobject import decref
 
 from pypy.module.cpyext.dictobject import PyDict_Check
 
@@ -252,7 +252,7 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        Py_DecRef(w_list.space, storage._elems[index])
+        decref(w_list.space, storage._elems[index])
         storage._elems[index] = make_ref(w_list.space, w_obj)
 
     def length(self, w_list):
@@ -264,9 +264,8 @@
         return storage._elems
 
     def getslice(self, w_list, start, stop, step, length):
-        #storage = self.unerase(w_list.lstorage)
-        raise oefmt(w_list.space.w_NotImplementedError,
-                    "settting a slice of a PySequence_Fast is not supported")
+        w_list.switch_to_object_strategy()
+        return w_list.strategy.getslice(w_list, start, stop, step, length)
 
     def getitems(self, w_list):
         # called when switching list strategy, so convert storage
@@ -389,5 +388,5 @@
 
     def __del__(self):
         for i in range(self._length):
-            Py_DecRef(self.space, self._elems[i])
+            decref(self.space, self._elems[i])
         lltype.free(self._elems, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.eval import (
     Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP
+from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
@@ -130,19 +130,19 @@
     def test_run_file(self, space, api):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
-        fp = fopen(str(filepath), "rb")
+        fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_ZeroDivisionError
         api.PyErr_Clear()
 
         # try again, but with a closed file
-        fp = fopen(str(filepath), "rb")
-        os.close(fileno(fp))
+        fp = c_fopen(str(filepath), "rb")
+        os.close(c_fileno(fp))
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -212,8 +212,9 @@
         assert type(x) is float
         assert x == -12.34
 
-    @pytest.mark.skipif(True, reason='realloc not fully implemented')
     def test_object_realloc(self):
+        if not self.runappdirect:
+            skip('no untranslated support for realloc')
         module = self.import_extension('foo', [
             ("realloctest", "METH_NOARGS",
              """
@@ -221,12 +222,11 @@
                  char *copy, *orig = PyObject_MALLOC(12);
                  memcpy(orig, "hello world", 12);
                  copy = PyObject_REALLOC(orig, 15);
+                 /* realloc() takes care of freeing orig, if changed */
                  if (copy == NULL)
                      Py_RETURN_NONE;
                  ret = PyBytes_FromStringAndSize(copy, 12);
-                 if (copy != orig)
-                     PyObject_Free(copy);
-                 PyObject_Free(orig);
+                 PyObject_Free(copy);
                  return ret;
              """)])
         x = module.realloctest()
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -78,6 +78,17 @@
         assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
         assert space.eq_w(w_t, space.wrap([1, 3, 5]))
 
+    def test_get_slice_fast(self, space, api):
+        w_t = space.wrap([1, 2, 3, 4, 5])
+        api.PySequence_Fast(w_t, "foo") # converts
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4]
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4]
+
+        assert api.PySequence_DelSlice(w_t, 1, 4) == 0
+        assert space.eq_w(w_t, space.wrap([1, 5]))
+        assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
+        assert space.eq_w(w_t, space.wrap([1, 3, 5]))
+
     def test_iter(self, space, api):
         w_t = space.wrap((1, 2))
         w_iter = api.PySeqIter_New(w_t)
@@ -226,18 +237,33 @@
         assert space.int_w(space.len(w_l)) == 10
 
 
-class XAppTestSequenceObject(AppTestCpythonExtensionBase):
-    def test_sequenceobject(self):
+class AppTestSequenceObject(AppTestCpythonExtensionBase):
+    def test_fast(self):
         module = self.import_extension('foo', [
             ("test_fast_sequence", "METH_VARARGS",
              """
-                PyObject * o = PyTuple_GetItem(args, 0);
+                int size, i;
+                PyTypeObject * common_type;
+                PyObject *foo, **objects;
+                PyObject * seq = PyTuple_GetItem(args, 0);
                 /* XXX assert it is a tuple */
-                PyObject *foo = PySequence_Fast(o, "some string");
-                PyObject ** res = PySequence_Fast_ITEMS(foo);
-                /* XXX do some kind of test on res */
-                /* XXX now what? who manages res's refcount? */
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "some string");
+                objects = PySequence_Fast_ITEMS(foo);
+                size = PySequence_Fast_GET_SIZE(seq);
+                common_type = size > 0 ? Py_TYPE(objects[0]) : NULL;
+                for (i = 1; i < size; ++i) {
+                    if (Py_TYPE(objects[i]) != common_type) {
+                        common_type = NULL;
+                        break;
+                    }
+                }
+                Py_DECREF(foo);
+                Py_DECREF(common_type);
                 return PyBool_FromLong(1);
              """)])
-        assert module.test_fast_sequence([1, 2, 3, 4])
+        s = [1, 2, 3, 4]
+        assert module.test_fast_sequence(s[0:-1])
+        assert module.test_fast_sequence(s[::-1])
 
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -47,6 +47,7 @@
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
+    from rpython.tool.error import offset2lineno
     bcindex = ord(bytecode.co_code[next_instr])
     opname = ""
     if 0 <= bcindex < len(opcode_method_names):
@@ -54,7 +55,8 @@
     name = bytecode.co_name
     if not name:
         name = ""
-    return (bytecode.co_filename, bytecode.co_firstlineno,
+    line = offset2lineno(bytecode, intmask(next_instr))
+    return (bytecode.co_filename, line,
             name, intmask(next_instr), opname)
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -19,6 +19,7 @@
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
         self.debug = True
+        self.track_resources = False
         self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -130,7 +130,7 @@
                 cls.module = str(udir.join('testownlib.dll'))
         else:
             subprocess.check_call(
-                'gcc testownlib.c -shared -fPIC -o testownlib.so',
+                'cc testownlib.c -shared -fPIC -o testownlib.so',
                 cwd=str(udir), shell=True)
             cls.module = str(udir.join('testownlib.so'))
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -852,9 +852,12 @@
     assert str(e2.value) == "foo0() takes no arguments (2 given)"
     assert str(e3.value) == "foo1() takes exactly one argument (0 given)"
     assert str(e4.value) == "foo1() takes exactly one argument (2 given)"
-    assert str(e5.value) == "foo2() takes exactly 2 arguments (0 given)"
-    assert str(e6.value) == "foo2() takes exactly 2 arguments (1 given)"
-    assert str(e7.value) == "foo2() takes exactly 2 arguments (3 given)"
+    assert str(e5.value) in ["foo2 expected 2 arguments, got 0",
+                             "foo2() takes exactly 2 arguments (0 given)"]
+    assert str(e6.value) in ["foo2 expected 2 arguments, got 1",
+                             "foo2() takes exactly 2 arguments (1 given)"]
+    assert str(e7.value) in ["foo2 expected 2 arguments, got 3",
+                             "foo2() takes exactly 2 arguments (3 given)"]
 
 def test_address_of_function():
     ffi = FFI()
@@ -1916,3 +1919,47 @@
     ffi.cdef("bool f(void);")
     lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
     assert lib.f() == 1
+
+def test_bool_in_cpp_2():
+    ffi = FFI()
+    ffi.cdef('int add(int a, int b);')
+    lib = verify(ffi, "test_bool_bug_cpp", '''
+        typedef bool _Bool;  /* there is a Windows header with this line */
+        int add(int a, int b)
+        {
+            return a + b;
+        }''', source_extension='.cpp')
+    c = lib.add(2, 3)
+    assert c == 5
+
+def test_struct_field_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[2]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+
+def test_function_arg_opaque():
+    py.test.skip("can currently declare a function with an opaque struct "
+                 "as argument, but AFAICT it's impossible to call it later")
+
+def test_function_returns_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a foo(int);")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_function_returns_opaque", "?")
+    assert str(e.value) == ("function foo: 'struct a' is used as result type,"
+                            " but is opaque")
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -133,6 +133,12 @@
         # You cannot assing character format codes as restype any longer
         raises(TypeError, setattr, f, "restype", "i")
 
+    def test_unicode_function_name(self):
+        f = dll[u'_testfunc_i_bhilfd']
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_int
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        assert result == 21
 
     def test_truncate_python_longs(self):
         f = dll._testfunc_i_bhilfd
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -159,7 +159,6 @@
         libraries=rtime.libraries
     )
     CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
@@ -233,7 +232,6 @@
 HAS_CLOCK_MONOTONIC = cConfig.CLOCK_MONOTONIC is not None
 HAS_MONOTONIC = (_WIN or _MACOSX or
                  (HAS_CLOCK_GETTIME and (HAS_CLOCK_HIGHRES or HAS_CLOCK_MONOTONIC)))
-clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
 
@@ -1030,7 +1028,10 @@
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
                 if rffi.cast(lltype.Signed, ret) != -1:
-                    cpu_time = float(tms.c_tms_utime + tms.c_tms_stime)
+                    cpu_time = float(rffi.cast(lltype.Signed,
+                                               tms.c_tms_utime) +
+                                     rffi.cast(lltype.Signed,
+                                               tms.c_tms_stime))
                     if w_info is not None:
                         _setinfo(space, w_info, "times()",
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
@@ -1038,7 +1039,7 @@
                     return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
         return clock(space)
 
-_clock = external('clock', [], clock_t)
+_clock = external('clock', [], rposix.CLOCK_T)
 def clock(space, w_info=None):
     """clock() -> floating point number
 
@@ -1052,7 +1053,7 @@
             pass
     value = _clock()
     # Is this casting correct?
-    if value == rffi.cast(clock_t, -1):
+    if intmask(value) == intmask(rffi.cast(rposix.CLOCK_T, -1)):
         raise oefmt(space.w_RuntimeError,
                     "the processor time used is not available or its value"
                     "cannot be represented")
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -930,6 +930,7 @@
         abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj))
 
 def type_get_dict(space, w_cls):
+    w_cls = _check(space, w_cls)
     from pypy.objspace.std.dictproxyobject import W_DictProxyObject
     w_dict = w_cls.getdict(space)
     if w_dict is None:
@@ -1287,7 +1288,8 @@
     cycle.append(candidate)
     cycle.reverse()
     names = [cls.getname(space) for cls in cycle]
-    raise OperationError(space.w_TypeError, space.wrap(
+    # Can't use oefmt() here, since names is a list of unicodes
+    raise OperationError(space.w_TypeError, space.newunicode(
         u"cycle among base classes: " + u' < '.join(names)))
 
 
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -143,3 +143,5 @@
     def is_w(self, obj1, obj2):
         return obj1 is obj2
 
+    def setitem(self, obj, key, value):
+        obj[key] = value
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-# hypothesis is used for test generation on untranslated jit tests
+# hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -401,6 +401,9 @@
 class __extend__(pairtype(SomeString, SomeTuple),
                  pairtype(SomeUnicodeString, SomeTuple)):
     def mod((s_string, s_tuple)):
+        if not s_string.is_constant():
+            raise AnnotatorError("string formatting requires a constant "
+                                 "string/unicode on the left of '%'")
         is_string = isinstance(s_string, SomeString)
         is_unicode = isinstance(s_string, SomeUnicodeString)
         assert is_string or is_unicode
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4623,6 +4623,14 @@
         a = self.RPythonAnnotator()
         a.build_types(main, [int])
 
+    def test_string_mod_nonconstant(self):
+        def f(x):
+            return x % 5
+        a = self.RPythonAnnotator()
+        e = py.test.raises(AnnotatorError, a.build_types, f, [str])
+        assert ('string formatting requires a constant string/unicode'
+                in str(e.value))
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -883,6 +883,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                               self.cpu.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -280,7 +280,7 @@
     concrete_type = '\x00'
 
     def __init__(self, basesize, itemsize, lendescr, flag, is_pure=False, concrete_type='\x00'):
-        self.basesize = basesize
+        self.basesize = basesize    # this includes +1 for STR
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
@@ -676,7 +676,7 @@
 
 def unpack_arraydescr(arraydescr):
     assert isinstance(arraydescr, ArrayDescr)
-    ofs = arraydescr.basesize
+    ofs = arraydescr.basesize    # this includes +1 for STR
     size = arraydescr.itemsize
     sign = arraydescr.is_item_signed()
     return size, ofs, sign
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -293,6 +293,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_load_or_indexed(op, op.getarg(0), op.getarg(1),
                                          itemsize, itemsize, basesize, NOT_SIGNED)
         elif opnum == rop.UNICODEGETITEM:
@@ -304,6 +305,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_store_or_indexed(op, op.getarg(0), op.getarg(1), op.getarg(2),
                                          itemsize, itemsize, basesize)
         elif opnum == rop.UNICODESETITEM:
diff --git a/rpython/jit/backend/llsupport/symbolic.py b/rpython/jit/backend/llsupport/symbolic.py
--- a/rpython/jit/backend/llsupport/symbolic.py
+++ b/rpython/jit/backend/llsupport/symbolic.py
@@ -29,7 +29,7 @@
 def get_array_token(T, translate_support_code):
     # T can be an array or a var-sized structure
     if translate_support_code:
-        basesize = llmemory.sizeof(T, 0)
+        basesize = llmemory.sizeof(T, 0)     # this includes +1 for STR
         if isinstance(T, lltype.Struct):
             SUBARRAY = getattr(T, T._arrayfld)
             itemsize = llmemory.sizeof(SUBARRAY.OF)
@@ -57,6 +57,7 @@
             assert carray.length.size == WORD
             ofs_length = before_array_part + carray.length.offset
         basesize = before_array_part + carray.items.offset
+        basesize += T._hints.get('extra_item_after_alloc', 0)  # +1 for STR
         carrayitem = ll2ctypes.get_ctypes_type(T.OF)
         itemsize = ctypes.sizeof(carrayitem)
     return basesize, itemsize, ofs_length
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -435,8 +435,10 @@
 def test_bytearray_descr():
     c0 = GcCache(False)
     descr = get_array_descr(c0, rstr.STR)   # for bytearray
+    # note that we get a basesize that has 1 extra byte for the final null char
+    # (only for STR)
     assert descr.flag == FLAG_UNSIGNED
-    assert descr.basesize == struct.calcsize("PP")         # hash, length
+    assert descr.basesize == struct.calcsize("PP") + 1     # hash, length, extra
     assert descr.lendescr.offset == struct.calcsize("P")   # hash
     assert not descr.is_array_of_pointers()
 
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -647,6 +647,9 @@
         """)
 
     def test_rewrite_assembler_newstr_newunicode(self):
+        # note: strdescr.basesize already contains the extra final character,
+        # so that's why newstr(14) is rounded up to 'basesize+15' and not
+        # 'basesize+16'.
         self.check_rewrite("""
             [i2]
             p0 = newstr(14)
@@ -657,12 +660,12 @@
         """, """
             [i2]
             p0 = call_malloc_nursery(                                \
-                      %(strdescr.basesize + 16 * strdescr.itemsize + \
+                      %(strdescr.basesize + 15 * strdescr.itemsize + \
                         unicodedescr.basesize + 10 * unicodedescr.itemsize)d)
             gc_store(p0, 0,  %(strdescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p0, %(strlendescr.offset)s, 14, %(strlendescr.field_size)s)
             gc_store(p0, 0,  0, %(strhashdescr.field_size)s)
-            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
+            p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 15 * strdescr.itemsize)d)
             gc_store(p1, 0,  %(unicodedescr.tid)d, %(tiddescr.field_size)s)
             gc_store(p1, %(unicodelendescr.offset)s, 10, %(unicodelendescr.field_size)s)
             gc_store(p1, 0,  0, %(unicodehashdescr.field_size)s)
@@ -1240,14 +1243,14 @@
         #               'i3 = gc_load_i(p0,i5,%(unicodedescr.itemsize)d)'],
         [True,  (4,),  'i3 = strgetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,1,'
-                       '%(strdescr.basesize)d,1)'],
+                       '%(strdescr.basesize-1)d,1)'],
         #[False, (4,),  'i3 = strgetitem(p0,i1)' '->'
-        #               'i5 = int_add(i1, %(strdescr.basesize)d);'
+        #               'i5 = int_add(i1, %(strdescr.basesize-1)d);'
         #               'i3 = gc_load_i(p0,i5,1)'],
         ## setitem str/unicode
         [True, (4,),  'i3 = strsetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,1,'
-                               '%(strdescr.basesize)d,1)'],
+                               '%(strdescr.basesize-1)d,1)'],
         [True, (2,4), 'i3 = unicodesetitem(p0,i1,0)' '->'
                       'i3 = gc_store_indexed(p0,i1,0,'
                                  '%(unicodedescr.itemsize)d,'
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -3,7 +3,7 @@
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
 from rpython.rlib.jit import promote, _get_virtualizable_token
-from rpython.rlib import jit_hooks, rposix
+from rpython.rlib import jit_hooks, rposix, rgc
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
 from rpython.jit.backend.detect_cpu import getcpuclass
@@ -11,7 +11,7 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 from rpython.config.config import ConfigError
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.rlib.rjitlog import rjitlog as jl
 
 
@@ -29,6 +29,7 @@
         # - floats neg and abs
         # - cast_int_to_float
         # - llexternal with macro=True
+        # - extra place for the zero after STR instances
 
         class BasicFrame(object):
             _virtualizable_ = ['i']
@@ -56,7 +57,7 @@
             return ("/home.py",0,0)
 
         jitdriver = JitDriver(greens = [],
-                              reds = ['total', 'frame', 'j'],
+                              reds = ['total', 'frame', 'prev_s', 'j'],
                               virtualizables = ['frame'],
                               get_location = get_location)
         def f(i, j):
@@ -68,9 +69,12 @@
             total = 0
             frame = Frame(i)
             j = float(j)
+            prev_s = rstr.mallocstr(16)

From pypy.commits at gmail.com  Tue Aug 16 05:20:56 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 16 Aug 2016 02:20:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Fix async with test,
 fixes timing for messages expected in assert
Message-ID: <57b2daf8.8aacc20a.ea595.9df1@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86214:ce0987e21403
Date: 2016-08-16 11:20 +0200
http://bitbucket.org/pypy/pypy/changeset/ce0987e21403/

Log:	Fix async with test, fixes timing for messages expected in assert

diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -44,5 +44,11 @@
 finally:
     loop.close()
 
-assert cor.res == "- coro 1: waiting for lock - coro 1: holding the lock - coro 2: waiting for lock - coro 1: releasing the lock - coro 2: holding the lock - coro 2: releasing the lock -"
+assert "coro 1: waiting for lock" in cor.res
+assert "coro 1: holding the lock" in cor.res
+assert "coro 1: releasing the lock" in cor.res
+assert "coro 2: waiting for lock" in cor.res
+assert "coro 2: holding the lock" in cor.res
+assert "coro 2: releasing the lock" in cor.res
+assert cor.res.find("coro 1: releasing the lock") < cor.res.find("coro 2: holding the lock")
         """

From pypy.commits at gmail.com  Tue Aug 16 05:56:28 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 16 Aug 2016 02:56:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: hex method for bytes and
 bytearray + test
Message-ID: <57b2e34c.0205c20a.a1731.ad04@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86215:75e21c6f7535
Date: 2016-08-16 11:55 +0200
http://bitbucket.org/pypy/pypy/changeset/75e21c6f7535/

Log:	hex method for bytes and bytearray + test

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -441,6 +441,9 @@
     def descr_copy(self, space):
         return self._new(self.data[:])
 
+    def descr_hex(self, space):
+        return _array_to_hexstring(space, self.data, len(self.data), True)
+
 
 # ____________________________________________________________
 # helpers for slow paths, moved out because they contain loops
@@ -494,15 +497,22 @@
 HEXDIGITS = "0123456789abcdef"
 PY_SIZE_T_MAX = 2**(rffi.sizeof(rffi.SIZE_T)*8)-1
 
-def _array_to_hexstring(space, buf):
-    length = buf.getlength()
+ at specialize.arg(3) # raw access
+def _array_to_hexstring(space, buf, len=0, rawaccess=False):
+    if rawaccess:
+        length = len
+    else:
+        length = buf.getlength()
     hexstring = StringBuilder(length*2)
 
     if length > PY_SIZE_T_MAX/2:
         raise OperationError(space.w_MemoryError)
 
     for i in range(length):
-        byte = ord(buf.getitem(i))
+        if rawaccess:
+            byte = ord(buf[i])
+        else:
+            byte = ord(buf.getitem(i))
         c = (byte >> 4 & 0xf)
         hexstring.append(HEXDIGITS[c])
         c = (byte & 0xf)
@@ -944,6 +954,12 @@
         of the specified width.  B is never truncated.
         """
 
+    def hex():
+        """B.hex() -> unicode
+        Return a string object containing two hexadecimal digits
+        for each byte in the instance B.
+        """
+
 
 W_BytearrayObject.typedef = TypeDef(
     "bytearray",
@@ -1093,6 +1109,8 @@
                        doc=BytearrayDocstrings.clear.__doc__),
     copy = interp2app(W_BytearrayObject.descr_copy,
                          doc=BytearrayDocstrings.copy.__doc__),
+    hex = interp2app(W_BytearrayObject.descr_hex,
+                           doc=BytearrayDocstrings.hex.__doc__),
 )
 W_BytearrayObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -394,6 +394,12 @@
         of the specified width. The string S is never truncated.
         """
 
+    def descr_hex(self, space):
+        """S.hex() -> string
+
+        Creates a hexadecimal string of the bytes object
+        """
+
 
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
@@ -648,6 +654,11 @@
     def descr_upper(self, space):
         return W_BytesObject(self._value.upper())
 
+    def descr_hex(self, space):
+        from pypy.objspace.std.bytearrayobject import _array_to_hexstring
+        return _array_to_hexstring(space, self.buffer_w(space, space.BUF_SIMPLE))
+
+
 
 def _create_list_from_bytes(value):
     # need this helper function to allow the jit to look inside and inline
@@ -827,6 +838,7 @@
 
     fromhex = interp2app(W_BytesObject.descr_fromhex, as_classmethod=True),
     maketrans = interp2app(W_BytesObject.descr_maketrans, as_classmethod=True),
+    hex = interp2app(W_BytesObject.descr_hex)
 )
 W_BytesObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -523,3 +523,7 @@
         result = bytearray.maketrans(b'abc', b'xyz')
         assert result == table
         assert type(result) is bytes
+
+    def test_hex(self):
+        assert bytearray(b'santa claus').hex() == "73616e746120636c617573"
+
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -870,3 +870,10 @@
             def __int__(self):
                 return 42
         raises(TypeError, bytes, A())
+
+    def test_hex(self):
+        assert bytes('santa claus', 'ascii').hex() == "73616e746120636c617573"
+        assert bytes([0x73,0x61,0x6e,0x74,0x61,0x20,0x63,0x6c,0x61,0x75,0x73]).hex() == \
+               "73616e746120636c617573"
+        assert bytes(64).hex() == "00"*64
+

From pypy.commits at gmail.com  Tue Aug 16 06:02:42 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 16 Aug 2016 03:02:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: 32bit translation issue,
 prebuilt long
Message-ID: <57b2e4c2.8bc71c0a.8c8f4.f1b0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86216:42637c644eab
Date: 2016-08-16 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/42637c644eab/

Log:	32bit translation issue, prebuilt long

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -3,6 +3,7 @@
 from rpython.rlib.objectmodel import (
     import_from_mixin, newlist_hint, resizelist_hint, specialize)
 from rpython.rlib.buffer import Buffer
+from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rstring import StringBuilder, ByteListBuilder
 from rpython.rlib.debug import check_list_of_chars
 from rpython.rtyper.lltypesystem import rffi
@@ -495,7 +496,7 @@
     return data
 
 HEXDIGITS = "0123456789abcdef"
-PY_SIZE_T_MAX = 2**(rffi.sizeof(rffi.SIZE_T)*8)-1
+PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8)-1)
 
 @specialize.arg(3) # raw access
 def _array_to_hexstring(space, buf, len=0, rawaccess=False):

From pypy.commits at gmail.com  Tue Aug 16 06:09:30 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 16 Aug 2016 03:09:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: missing argument to operation
 error
Message-ID: <57b2e65a.81a2c20a.2c7cc.b383@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86217:3045edf9288b
Date: 2016-08-16 12:05 +0200
http://bitbucket.org/pypy/pypy/changeset/3045edf9288b/

Log:	missing argument to operation error

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -507,7 +507,7 @@
     hexstring = StringBuilder(length*2)
 
     if length > PY_SIZE_T_MAX/2:
-        raise OperationError(space.w_MemoryError)
+        raise OperationError(space.w_MemoryError, space.w_None)
 
     for i in range(length):
         if rawaccess:

From pypy.commits at gmail.com  Tue Aug 16 06:12:00 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 16 Aug 2016 03:12:00 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: further implementing vec
 unpack/pack on ppc
Message-ID: <57b2e6f0.85c11c0a.b917.ee10@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86218:d41babc2d791
Date: 2016-08-16 12:11 +0200
http://bitbucket.org/pypy/pypy/changeset/d41babc2d791/

Log:	further implementing vec unpack/pack on ppc

diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -660,7 +660,7 @@
     # splat high of A, and high of B
     xxspltdh = XX3_splat(60, XO13=10, OE=0, DM=0b11)
     # generic splat
-    xxspltd = XX3_splat(60, XO13=10, OE=0)
+    xxpermdi = XX3_splat(60, XO13=10, OE=0)
 
     xxlxor = XX3(60, XO9=154)
     xxlor = XX3(60, XO9=146)
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -21,6 +21,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.jit.codewriter import longlong
 from rpython.jit.backend.ppc.detect_feature import detect_vsx
+from rpython.rlib.objectmodel import always_inline
 
 def not_implemented(msg):
     msg = '[ppc/vector_ext] %s\n' % msg
@@ -28,6 +29,24 @@
         llop.debug_print(lltype.Void, msg)
     raise NotImplementedError(msg)
 
+ at always_inline
+def permi(v1, v2):
+    """ permute immediate for big and little endian """
+    # if v1 == 0 unpacks index 0 of param 1
+    # if v1 == 1 unpacks index 1 of param 1
+    # if v2 == 0 unpacks index 0 of param 2
+    # if v2 == 1 unpacks index 1 of param 2
+    mask = 0
+    if IS_BIG_ENDIAN:
+        not_implemented("no big endian support (yet)")
+    else:
+        if v1 == 0: mask |= 0b01
+        if v1 == 1: mask |= 0b00
+        if v2 == 0: mask |= 0b10
+        if v2 == 1: mask |= 0b00
+    return mask
+
+
 def flush_vec_cc(asm, regalloc, condition, size, result_loc):
     # After emitting an instruction that leaves a boolean result in
     # a condition code (cc), call this.  In the common case, result_loc
@@ -448,9 +467,12 @@
         size = op.bytesize
         if size == 8:
             if resultloc.is_vector_reg(): # vector <- reg
-                self.mc.load_imm(r.SCRATCH, PARAM_SAVE_AREA_OFFSET)
+                self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
                 self.mc.stvx(vector, r.SCRATCH2.value, r.SP.value)
-                self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*residx)
+                idx = residx
+                if not IS_BIG_ENDIAN:
+                    idx = 1 - idx
+                self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx)
                 self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
             else:
                 not_implemented("64 bit float")
@@ -463,29 +485,32 @@
 
     def emit_vec_unpack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
-        resloc, srcloc, idxloc, countloc = arglocs
+        resloc, srcloc, idxloc, countloc, sizeloc = arglocs
         idx = idxloc.value
         res = resloc.value
         src = srcloc.value
-        size = op.bytesize
+        size = sizeloc.value
         count = countloc.value
         if count == 1:
             assert srcloc.is_vector_reg()
             assert not resloc.is_vector_reg()
             off = PARAM_SAVE_AREA_OFFSET
             self.mc.load_imm(r.SCRATCH2, off)
-            off = off + size*idx
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
+            off = off + size * idx
             if size == 8:
                 self.mc.load(res, r.SP.value, off+size*idx)
+                return
             elif size == 4:
                 self.mc.lwa(res, r.SP.value, off)
+                return
             elif size == 2:
                 self.mc.lha(res, r.SP.value, off)
+                return
             elif size == 1:
                 self.mc.lbz(res, r.SP.value, off)
                 self.mc.extsb(res, res)
-            return
+                return
 
         not_implemented("%d bit integer, count %d" % \
                        (size*8, count))
@@ -500,42 +525,44 @@
         residx = residxloc.value
         srcidx = srcidxloc.value
         size = op.bytesize
-        assert size == 8
         # srcloc is always a floating point register f, this means it is
         # vsr[0] == valueof(f)
         if srcidx == 0:
             if residx == 0:
-                # r = (s[0], r[1])
-                if IS_BIG_ENDIAN:
-                    self.mc.xxspltd(res, src, vec, 0b10)
-                else:
-                    self.mc.xxspltd(res, src, vec, 0b01)
+                # r = (s[0], v[1])
+                self.mc.xxpermdi(res, src, vec, permi(0,1))
             else:
                 assert residx == 1
-                # r = (r[0], s[0])
-                if IS_BIG_ENDIAN:
-                    self.mc.xxspltd(res, vec, src, 0b00)
-                else:
-                    self.mc.xxspltd(res, vec, src, 0b11)
+                # r = (v[0], s[0])
+                self.mc.xxpermdi(res, vec, src, permi(1,1))
         else:
             assert srcidx == 1
             if residx == 0:
-                # r = (s[1], r[1])
-                if IS_BIG_ENDIAN:
-                    self.mc.xxspltd(res, src, vec, 0b11)
-                else:
-                    self.mc.xxspltd(res, src, vec, 0b00)
+                # r = (s[1], v[1])
+                self.mc.xxpermdi(res, src, vec, permi(1,1))
             else:
                 assert residx == 1
-                # r = (r[0], s[1])
-                if IS_BIG_ENDIAN:
-                    self.mc.xxspltd(res, vec, src, 0b10)
-                else:
-                    self.mc.xxspltd(res, vec, src, 0b01)
+                # r = (v[0], s[1])
+                self.mc.xxpermdi(res, vec, src, permi(0,1))
 
     def emit_vec_unpack_f(self, op, arglocs, regalloc):
-        resloc, srcloc, idxloc, countloc = arglocs
-        self.emit_vec_pack_f(op, [resloc, srcloc, srcloc, imm(0), idxloc, countloc], regalloc)
+        assert isinstance(op, VectorOp)
+        resloc, srcloc, srcidxloc, countloc = arglocs
+        res = resloc.value
+        src = srcloc.value
+        srcidx = srcidxloc.value
+        size = op.bytesize
+        # srcloc is always a floating point register f, this means it is
+        # vsr[0] == valueof(f)
+        if srcidx == 0:
+            # r = (s[0], s[1])
+            self.mc.xxpermdi(res, src, src, permi(0,1))
+            return
+        else:
+            # r = (s[1], s[0])
+            self.mc.xxpermdi(res, src, src, permi(1,0))
+            return
+        not_implemented("unpack for combination src %d -> res %d" % (srcidx, residx))
 
     def emit_vec_cast_float_to_int(self, op, arglocs, regalloc):
         res, l0 = arglocs
@@ -700,7 +727,10 @@
         assert not arg.is_vector()
         srcloc = self.ensure_reg(arg)
         vloc = self.ensure_vector_reg(op.getarg(0))
-        resloc = self.force_allocate_vector_reg(op)
+        if op.is_vector():
+            resloc = self.force_allocate_vector_reg(op)
+        else:
+            resloc = self.force_allocate_reg(op)
         residx = index.value # where to put it in result?
         srcidx = 0
         return [resloc, vloc, srcloc, imm(residx), imm(srcidx), imm(count.value)]
@@ -722,11 +752,13 @@
         assert isinstance(count, ConstInt)
         arg = op.getarg(0)
         if arg.is_vector():
-            srcloc = self.ensure_vector_reg(op.getarg(0))
+            srcloc = self.ensure_vector_reg(arg)
         else:
-            srcloc = self.ensure_reg(op.getarg(0))
+            # unpack
+            srcloc = self.ensure_reg(arg0)
+        size = arg.bytesize
         resloc = self.force_allocate_reg(op)
-        return [resloc, srcloc, imm(index.value), imm(count.value)]
+        return [resloc, srcloc, imm(index.value), imm(count.value), imm(size)]
 
     def expand_float(self, size, box):
         adr = self.assembler.datablockwrapper.malloc_aligned(16, 16)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -743,7 +743,7 @@
             pack.append('%s = vec_%s()' % (v, suffix))
             for i,val in enumerate(vals):
                 args_values.append(val)
-                f = newvar('f')
+                f = newvar(suffix)
                 args.append(f)
                 count = 1
                 # create a new variable
@@ -789,6 +789,7 @@
         #
         looptoken = JitCellToken()
         cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+        import pdb; pdb.set_trace()
         deadframe = cpu.execute_token(looptoken, *args_values)
         print(source)
         if float:
@@ -796,26 +797,30 @@
         else:
             return cpu.get_int_value(deadframe, 0)
 
-    def test_unpack(self):
+    def test_unpack_f(self):
         # double unpack
         assert self.run_unpack("f{f} = vec_unpack_f({x}, 0, 1)",
-                               "[2xf64]", {'x': (1.2,-1)}) == 1.2
+                               "[2xf64]", {'x': (1.2,-1.0)}) == 1.2
         assert self.run_unpack("f{f} = vec_unpack_f({x}, 1, 1)",
                                "[2xf64]", {'x': (50.33,4321.0)}) == 4321.0
+    def test_unpack_i64(self):
         # int64
         assert self.run_unpack("i{i} = vec_unpack_i({x}, 0, 1)",
                                "[2xi64]", {'x': (11,12)}, float=False) == 11
         assert self.run_unpack("i{i} = vec_unpack_i({x}, 1, 1)",
                                "[2xi64]", {'x': (14,15)}, float=False) == 15
 
-        ## integer unpack (byte)
+    def test_unpack_i(self):
         for i in range(16):
+            # i8
             op = "i{i} = vec_unpack_i({x}, %d, 1)" % i
             assert self.run_unpack(op, "[16xi8]", {'x': [127,1]*8}, float=False) == \
                    (127 if i%2==0 else 1)
+            # i16
             if i < 8:
                 assert self.run_unpack(op, "[8xi16]", {'x': [2**15-1,0]*4}, float=False) == \
                        (2**15-1 if i%2==0 else 0)
+            # i32
             if i < 4:
                 assert self.run_unpack(op, "[4xi32]", {'x': [2**31-1,0]*4}, float=False) == \
                        (2**31-1 if i%2==0 else 0)

From pypy.commits at gmail.com  Tue Aug 16 06:56:35 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 16 Aug 2016 03:56:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: encode the jitdrivers name in
 start_trace, jitlog version bump
Message-ID: <57b2f163.497bc20a.13214.c014@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r86219:5847abf293f8
Date: 2016-08-04 16:32 +0200
http://bitbucket.org/pypy/pypy/changeset/5847abf293f8/

Log:	encode the jitdrivers name in start_trace, jitlog version bump

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1051,8 +1051,9 @@
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     #
+    jd_name = jitdriver_sd.jitdriver.name
     metainterp_sd.jitlog.start_new_trace(metainterp_sd,
-            faildescr=resumekey, entry_bridge=False)
+            faildescr=resumekey, entry_bridge=False, jd_name=jd_name)
     #
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 1
+JITLOG_VERSION = 2
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -331,7 +331,7 @@
     def finish(self):
         jitlog_teardown()
 
-    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False):
+    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False, jd_name=""):
         # even if the logger is not enabled, increment the trace id
         self.trace_id += 1
         if not jitlog_enabled():
@@ -345,6 +345,7 @@
         else:
             content.append(encode_str('loop'))
             content.append(encode_le_addr(int(entry_bridge)))
+        content.append(encode_str(jd_name))
         self._write_marked(MARK_START_TRACE, ''.join(content))
 
     def trace_aborted(self):

From pypy.commits at gmail.com  Tue Aug 16 06:56:37 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 16 Aug 2016 03:56:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: jitlog encode fail arguments (how could
 I forget that :)
Message-ID: <57b2f165.8aacc20a.ea595.c57c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r86220:67398046dd30
Date: 2016-08-05 11:17 +0200
http://bitbucket.org/pypy/pypy/changeset/67398046dd30/

Log:	jitlog encode fail arguments (how could I forget that :) ppc
	misaligned_is_fine = Ture

diff --git a/rpython/rlib/rawstorage.py b/rpython/rlib/rawstorage.py
--- a/rpython/rlib/rawstorage.py
+++ b/rpython/rlib/rawstorage.py
@@ -48,7 +48,8 @@
 try:
     cpuname = detect_cpu.autodetect()
     misaligned_is_fine = cpuname.startswith('x86') or \
-                         cpuname.startswith('s390x')
+                         cpuname.startswith('s390x') or \
+                         cpuname.startswith('ppc')
     del cpuname
 except detect_cpu.ProcessorAutodetectError:
     misaligned_is_fine = False
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -509,7 +509,9 @@
         """ an operation is written as follows:
             <marker> <opid (16 bit)> \
                      <len (32 bit)> \
-                     <res_val>,<arg_0>,...,<arg_n>,<descr>
+                     <res_val>,<arg_0>,...,<arg_n> \
+                     <descr>
+                     <failarg_0>,...<failarg_n>
             The marker indicates if the last argument is
             a descr or a normal argument.
         """
@@ -518,16 +520,21 @@
         le_opnum = encode_le_16bit(op.getopnum())
         str_res = self.var_to_str(op)
         line = ','.join([str_res] + str_args)
+        failargslist = op.getfailargs()
+        failargs = ''
+        if failargslist:
+            failargs = ','.join([self.var_to_str(farg) for farg in failargslist])
+        #
         if descr:
             descr_str = descr.repr_of_descr()
             line = line + ',' + descr_str
             string = encode_str(line)
             descr_number = compute_unique_id(descr)
             le_descr_number = encode_le_addr(descr_number)
-            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number
+            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number + encode_str(failargs)
         else:
             string = encode_str(line)
-            return MARK_RESOP, le_opnum + string
+            return MARK_RESOP, le_opnum + string + encode_str(failargs)
 
 
     def write_core_dump(self, operations, i, op, ops_offset):
@@ -579,6 +586,8 @@
         return ''.join(dump)
 
     def var_to_str(self, arg):
+        if arg is None:
+            return '-'
         try:
             mv = self.memo[arg]
         except KeyError:

From pypy.commits at gmail.com  Tue Aug 16 06:56:38 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 16 Aug 2016 03:56:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove change only needed by ppc vector
 ext branch
Message-ID: <57b2f166.54bc1c0a.2469c.0323@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r86221:c2da100e9466
Date: 2016-08-16 12:55 +0200
http://bitbucket.org/pypy/pypy/changeset/c2da100e9466/

Log:	remove change only needed by ppc vector ext branch

diff --git a/rpython/rlib/rawstorage.py b/rpython/rlib/rawstorage.py
--- a/rpython/rlib/rawstorage.py
+++ b/rpython/rlib/rawstorage.py
@@ -48,8 +48,7 @@
 try:
     cpuname = detect_cpu.autodetect()
     misaligned_is_fine = cpuname.startswith('x86') or \
-                         cpuname.startswith('s390x') or \
-                         cpuname.startswith('ppc')
+                         cpuname.startswith('s390x')
     del cpuname
 except detect_cpu.ProcessorAutodetectError:
     misaligned_is_fine = False

From pypy.commits at gmail.com  Tue Aug 16 10:19:11 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 16 Aug 2016 07:19:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Fix async with test,
 coroutines can assign lock in any order,
 but release of one lock has to appear before hold of the other lock
Message-ID: <57b320df.c19d1c0a.2bf58.5878@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86222:034dc2c2ce1c
Date: 2016-08-16 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/034dc2c2ce1c/

Log:	Fix async with test, coroutines can assign lock in any order, but
	release of one lock has to appear before hold of the other lock

diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -50,5 +50,6 @@
 assert "coro 2: waiting for lock" in cor.res
 assert "coro 2: holding the lock" in cor.res
 assert "coro 2: releasing the lock" in cor.res
-assert cor.res.find("coro 1: releasing the lock") < cor.res.find("coro 2: holding the lock")
+assert cor.res.find("coro 1: releasing the lock") < cor.res.find("coro 2: holding the lock") or \
+cor.res.find("coro 2: releasing the lock") < cor.res.find("coro 1: holding the lock")
         """

From pypy.commits at gmail.com  Tue Aug 16 10:23:11 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 16 Aug 2016 07:23:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Add description to
 test_asynchronous_context_managers
Message-ID: <57b321cf.d4e01c0a.bbd9.5040@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86223:bd2d21b70e51
Date: 2016-08-16 16:22 +0200
http://bitbucket.org/pypy/pypy/changeset/bd2d21b70e51/

Log:	Add description to test_asynchronous_context_managers

diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -20,6 +20,9 @@
         """
     
     def test_asynchronous_context_managers(self):
+        # it is important that "releasing lock A" happens before "holding lock B"
+        # or the other way around, but it is not allowed that both coroutines
+        # hold the lock at the same time
         """
 import encodings.idna
 import asyncio

From pypy.commits at gmail.com  Tue Aug 16 14:06:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:20 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: inheritable fds: minimal start
Message-ID: <57b3561c.c70a1c0a.bb180.a2c8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5671:f6fcf7c91d23
Date: 2016-08-16 17:29 +0200
http://bitbucket.org/pypy/extradoc/changeset/f6fcf7c91d23/

Log:	inheritable fds: minimal start

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -45,6 +45,10 @@
 
 * Newly created file descriptors are non-inheritable (PEP 446)
 
+  - added rposix.{set,get}_inheritable(), used it as a quick hack
+    inside interp_posix.pipe(), needed otherwise subprocess.Popen()
+    deadlocks
+
 * The marshal format has been made more compact and efficient
 
 * enum: Support for enumeration types (PEP 435).

From pypy.commits at gmail.com  Tue Aug 16 14:06:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: _imp.load_dynamic => _imp.create_dynamic
Message-ID: <57b35621.88cb1c0a.34113.69f4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86224:e633a352e2f6
Date: 2016-08-16 12:18 +0200
http://bitbucket.org/pypy/pypy/changeset/e633a352e2f6/

Log:	_imp.load_dynamic => _imp.create_dynamic

diff --git a/pypy/module/imp/__init__.py b/pypy/module/imp/__init__.py
--- a/pypy/module/imp/__init__.py
+++ b/pypy/module/imp/__init__.py
@@ -12,7 +12,7 @@
 
         'get_magic':       'interp_imp.get_magic',
         'get_tag':         'interp_imp.get_tag',
-        'load_dynamic':    'interp_imp.load_dynamic',
+        'create_dynamic':  'interp_imp.create_dynamic',
         'create_builtin':  'interp_imp.create_builtin',
         'init_frozen':     'interp_imp.init_frozen',
         'is_builtin':      'interp_imp.is_builtin',
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -117,6 +117,13 @@
         space.sys.setmodule(w_mod)
     return w_mod
 
+def load_c_extension(space, filename, modulename):
+    from pypy.module.cpyext.api import load_extension_module
+    log_pyverbose(space, 1, "import %s # from %s\n" %
+                  (modulename, filename))
+    load_extension_module(space, filename, modulename)
+    # NB. cpyext.api.load_extension_module() can also delegate to _cffi_backend
+
 # __________________________________________________________________
 #
 # import lock, to prevent two threads from running module-level code in
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -50,17 +50,13 @@
         fd = space.int_w(space.call_method(w_iobase, 'fileno'))
         return streamio.fdopen_as_stream(fd, filemode)
 
- at unwrap_spec(filename='fsencode')
-def load_dynamic(space, w_modulename, filename, w_file=None):
+def create_dynamic(space, w_spec, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
-
-    # the next line is mandatory to init cpyext
-    space.getbuiltinmodule("cpyext")
-
-    from pypy.module.cpyext.api import load_extension_module
-    load_extension_module(space, filename, space.str_w(w_modulename))
-
+    w_modulename = space.getattr(w_spec, space.wrap("name"))
+    w_path = space.getattr(w_spec, space.wrap("origin"))
+    filename = space.fsencode_w(w_path)
+    importing.load_c_extension(space, filename, space.str_w(w_modulename))
     return importing.check_sys_modules(space, w_modulename)
 
 def create_builtin(space, w_spec):
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -54,11 +54,13 @@
         finally:
             del sys.path[0]
 
-    def test_load_dynamic(self):
+    def test_create_dynamic(self):
         import imp
-        raises(ImportError, imp.load_dynamic, 'foo', 'bar')
-        raises(ImportError, imp.load_dynamic, 'foo', 'bar',
-               open(self.file_module))
+        class FakeSpec:
+            name = 'foo'
+            origin = 'this/path/does/not/exist'
+        raises(ImportError, imp.create_dynamic, FakeSpec())
+        raises(ImportError, imp.create_dynamic, FakeSpec(), "unused")
 
     def test_suffixes(self):
         import imp

From pypy.commits at gmail.com  Tue Aug 16 14:06:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: Silence (reasonable) gcc warnings and
 copy the style of cpython 2.7
Message-ID: <57b35622.41261c0a.9a39.a30f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86225:074e4ef9eb3a
Date: 2016-08-16 13:23 +0200
http://bitbucket.org/pypy/pypy/changeset/074e4ef9eb3a/

Log:	Silence (reasonable) gcc warnings and copy the style of cpython 2.7

diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
@@ -268,22 +268,26 @@
     min = 0;
     max = haystacksize;
 
-    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
+    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
         if (value < haystack[pos].uniseq) {
-            if (max == pos) break;
-            else max = pos;
+            if (max != pos) {
+                max = pos;
+                continue;
+            }
         }
         else if (value > haystack[pos].uniseq) {
-            if (min == pos) break;
-            else min = pos;
+            if (min != pos) {
+                min = pos;
+                continue;
+            }
         }
-        else
-            break;
+        break;
+    }
 
-        if (value == haystack[pos].uniseq)
-            return haystack[pos].code;
-        else
-            return DBCINV;
+    if (value == haystack[pos].uniseq) {
+        return haystack[pos].code;
+    }
+    return DBCINV;
 }
 #endif
 

From pypy.commits at gmail.com  Tue Aug 16 14:06:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix docstring
Message-ID: <57b35624.8a13c20a.21c51.5eb9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86226:2006ae854789
Date: 2016-08-16 13:23 +0200
http://bitbucket.org/pypy/pypy/changeset/2006ae854789/

Log:	fix docstring

diff --git a/pypy/module/imp/__init__.py b/pypy/module/imp/__init__.py
--- a/pypy/module/imp/__init__.py
+++ b/pypy/module/imp/__init__.py
@@ -2,8 +2,7 @@
 
 class Module(MixedModule):
     """
-    This module provides the components needed to build your own
-    __import__ function.
+    (Extremely) low-level import machinery bits as used by importlib and imp.
     """
     applevel_name = '_imp'
 

From pypy.commits at gmail.com  Tue Aug 16 14:06:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: slice.indices() is rewritten,
 duplicating functionality instead of reusing it
Message-ID: <57b35626.eeb8c20a.9231a.6014@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86227:5e6b9a703f80
Date: 2016-08-16 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/5e6b9a703f80/

Log:	slice.indices() is rewritten, duplicating functionality instead of
	reusing it

diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -153,10 +153,12 @@
         S. Out of bounds indices are clipped in a manner consistent with the
         handling of normal slices.
         """
-        length = space.getindex_w(w_length, space.w_OverflowError)
-        start, stop, step = self.indices3(space, length)
-        return space.newtuple([space.wrap(start), space.wrap(stop),
-                               space.wrap(step)])
+        # like CPython 3.5, we duplicate this whole functionality for
+        # this rarely-used method instead of using the existing logic
+        # in indices3(), just to support 'slice(a,b,c).indices(d)' where
+        # all of a, b, c and d are very large integers.
+        return app_indices(space, self.w_start, self.w_stop,
+                           self.w_step, w_length)
 
 
 def slicewprop(name):
@@ -248,3 +250,68 @@
     elif start > stop:
         start = stop
     return start, stop
+
+
+app = gateway.applevel("""
+    from operator import index
+
+    def evaluate_slice_index(x):
+        try:
+            return index(x)
+        except TypeError:
+            raise TypeError("slice indices must be integers or "
+                            "None or have an __index__ method")
+
+    def _getlongindices(start, stop, step, length):
+        if step is None:
+            step = 1
+        else:
+            step = evaluate_slice_index(step)
+            if step == 0:
+                raise ValueError("slice step cannot be zero")
+
+        # Find lower and upper bounds for start and stop.
+        if step < 0:
+            lower = -1
+            upper = length - 1
+        else:
+            lower = 0
+            upper = length
+
+        # Compute start.
+        if start is None:
+            start = upper if step < 0 else lower
+        else:
+            start = evaluate_slice_index(start)
+            if start < 0:
+                start += length
+                if start < lower:
+                    start = lower
+            else:
+                if start > upper:
+                    start = upper
+
+        # Compute stop.
+        if stop is None:
+            stop = lower if step < 0 else upper
+        else:
+            stop = evaluate_slice_index(stop)
+            if stop < 0:
+                stop += length
+                if stop < lower:
+                    stop = lower
+            else:
+                if stop > upper:
+                    stop = upper
+
+        return (start, stop, step)
+
+    def indices(start, stop, step, length):
+        length = index(length)
+        if length < 0:
+            raise ValueError("length should not be negative")
+        return _getlongindices(start, stop, step, length)
+
+""", filename=__file__)
+
+app_indices = app.interphook("indices")
diff --git a/pypy/objspace/std/test/test_sliceobject.py b/pypy/objspace/std/test/test_sliceobject.py
--- a/pypy/objspace/std/test/test_sliceobject.py
+++ b/pypy/objspace/std/test/test_sliceobject.py
@@ -111,11 +111,11 @@
         assert slice(-2 ** 200, -2 ** 100, 1).indices(1000) == (0, 0, 1)
         assert slice(2 ** 100, 0, -1).indices(1000) == (999, 0, -1)
         assert slice(2 ** 100, -2 ** 100, -1).indices(1000) == (999, -1, -1)
-        start, stop, step = slice(0, 1000, 2 ** 200).indices(1000)
-        assert start == 0
-        assert stop == 1000
-        assert step >= 1000
-        raises(OverflowError, "slice(0, 1000, 1).indices(2 ** 100)")
+        assert slice(0, 1000, 2 ** 200).indices(1000) == (0, 1000, 2 ** 200)
+        assert slice(0, 1000, 1).indices(2 ** 100) == (0, 1000, 1)
 
     def test_reduce(self):
         assert slice(1, 2, 3).__reduce__() == (slice, (1, 2, 3))
+
+    def test_indices_negative_length(self):
+        raises(ValueError, "slice(0, 1000, 1).indices(-1)")

From pypy.commits at gmail.com  Tue Aug 16 14:06:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Probably fix a compiler warning
Message-ID: <57b35628.469d1c0a.69927.a1c9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86228:58283fd10232
Date: 2016-08-16 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/58283fd10232/

Log:	Probably fix a compiler warning

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -96,12 +96,15 @@
             return 0;
         }
     ''',]
+    post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);']
 else:
     separate_module_sources = []
+    post_include_bits = []
     includes=['errno.h','stdio.h']
 errno_eci = ExternalCompilationInfo(
     includes=includes,
     separate_module_sources=separate_module_sources,
+    post_include_bits=post_include_bits,
 )
 
 # Direct getters/setters, don't use directly!

From pypy.commits at gmail.com  Tue Aug 16 14:06:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add rposix.{get, set}_inheritable(),
 needed by Python 3.5
Message-ID: <57b35629.09afc20a.90904.637f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86229:5066ac85f54d
Date: 2016-08-16 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/5066ac85f54d/

Log:	Add rposix.{get,set}_inheritable(), needed by Python 3.5

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -2049,3 +2049,40 @@
     def mknodat(path, mode, device, dir_fd=AT_FDCWD):
         error = c_mknodat(dir_fd, path, mode, device)
         handle_posix_error('mknodat', error)
+
+
+eci_inheritable = eci.merge(ExternalCompilationInfo(
+    separate_module_sources=["""
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable)
+{
+    /* XXX minimal impl. XXX */
+    int request = inheritable ? FIONCLEX : FIOCLEX;
+    return ioctl(fd, request, NULL);
+}
+RPY_EXTERN
+int rpy_get_inheritable(int fd)
+{
+    int flags = fcntl(fd, F_GETFD, 0);
+    if (flags == -1)
+        return -1;
+    return !(flags & FD_CLOEXEC);
+}
+    """],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+
+def set_inheritable(fd, inheritable):
+    error = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', error)
+
+def get_inheritable(fd):
+    res = c_get_inheritable(fd)
+    res = handle_posix_error('get_inheritable', res)
+    return res != 0
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -572,3 +572,12 @@
         os.close(dirfd)
     assert tmpdir.join('file').check(exists=False)
     assert tmpdir.join('file2').check(exists=True)
+
+def test_set_inheritable():
+    fd1, fd2 = os.pipe()
+    rposix.set_inheritable(fd1, True)
+    assert rposix.get_inheritable(fd1) == True
+    rposix.set_inheritable(fd1, False)
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)

From pypy.commits at gmail.com  Tue Aug 16 14:06:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add rposix.{get, set}_inheritable(),
 needed by Python 3.5
Message-ID: <57b3562b.82cbc20a.ac131.5892@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86230:60a94fdfc827
Date: 2016-08-16 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/60a94fdfc827/

Log:	Add rposix.{get,set}_inheritable(), needed by Python 3.5

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -2046,3 +2046,40 @@
     def mknodat(path, mode, device, dir_fd=AT_FDCWD):
         error = c_mknodat(dir_fd, path, mode, device)
         handle_posix_error('mknodat', error)
+
+
+eci_inheritable = eci.merge(ExternalCompilationInfo(
+    separate_module_sources=["""
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable)
+{
+    /* XXX minimal impl. XXX */
+    int request = inheritable ? FIONCLEX : FIOCLEX;
+    return ioctl(fd, request, NULL);
+}
+RPY_EXTERN
+int rpy_get_inheritable(int fd)
+{
+    int flags = fcntl(fd, F_GETFD, 0);
+    if (flags == -1)
+        return -1;
+    return !(flags & FD_CLOEXEC);
+}
+    """],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+
+def set_inheritable(fd, inheritable):
+    error = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', error)
+
+def get_inheritable(fd):
+    res = c_get_inheritable(fd)
+    res = handle_posix_error('get_inheritable', res)
+    return res != 0
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -572,3 +572,12 @@
         os.close(dirfd)
     assert tmpdir.join('file').check(exists=False)
     assert tmpdir.join('file2').check(exists=True)
+
+def test_set_inheritable():
+    fd1, fd2 = os.pipe()
+    rposix.set_inheritable(fd1, True)
+    assert rposix.get_inheritable(fd1) == True
+    rposix.set_inheritable(fd1, False)
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)

From pypy.commits at gmail.com  Tue Aug 16 14:06:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Quick fix,
 hopefully fixing issues of subprocess.Popen() in a translated
Message-ID: <57b3562d.8bc71c0a.8c8f4.ad6b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86231:2b797c855cb4
Date: 2016-08-16 17:08 +0200
http://bitbucket.org/pypy/pypy/changeset/2b797c855cb4/

Log:	Quick fix, hopefully fixing issues of subprocess.Popen() in a
	translated pypy

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -894,6 +894,9 @@
         fd1, fd2 = os.pipe()
     except OSError as e:
         raise wrap_oserror(space, e)
+    # XXX later, use rposix.pipe2() if available!
+    rposix.set_inheritable(fd1, False)
+    rposix.set_inheritable(fd2, False)
     return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),

From pypy.commits at gmail.com  Tue Aug 16 14:06:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <57b35630.031dc20a.de9d1.6a6d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86233:b7fe3a8d6872
Date: 2016-08-16 20:05 +0200
http://bitbucket.org/pypy/pypy/changeset/b7fe3a8d6872/

Log:	merge heads

diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
@@ -268,22 +268,26 @@
     min = 0;
     max = haystacksize;
 
-    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
+    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
         if (value < haystack[pos].uniseq) {
-            if (max == pos) break;
-            else max = pos;
+            if (max != pos) {
+                max = pos;
+                continue;
+            }
         }
         else if (value > haystack[pos].uniseq) {
-            if (min == pos) break;
-            else min = pos;
+            if (min != pos) {
+                min = pos;
+                continue;
+            }
         }
-        else
-            break;
+        break;
+    }
 
-        if (value == haystack[pos].uniseq)
-            return haystack[pos].code;
-        else
-            return DBCINV;
+    if (value == haystack[pos].uniseq) {
+        return haystack[pos].code;
+    }
+    return DBCINV;
 }
 #endif
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -96,12 +96,15 @@
             return 0;
         }
     ''',]
+    post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);']
 else:
     separate_module_sources = []
+    post_include_bits = []
     includes=['errno.h','stdio.h']
 errno_eci = ExternalCompilationInfo(
     includes=includes,
     separate_module_sources=separate_module_sources,
+    post_include_bits=post_include_bits,
 )
 
 # Direct getters/setters, don't use directly!
@@ -2046,3 +2049,40 @@
     def mknodat(path, mode, device, dir_fd=AT_FDCWD):
         error = c_mknodat(dir_fd, path, mode, device)
         handle_posix_error('mknodat', error)
+
+
+eci_inheritable = eci.merge(ExternalCompilationInfo(
+    separate_module_sources=["""
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable)
+{
+    /* XXX minimal impl. XXX */
+    int request = inheritable ? FIONCLEX : FIOCLEX;
+    return ioctl(fd, request, NULL);
+}
+RPY_EXTERN
+int rpy_get_inheritable(int fd)
+{
+    int flags = fcntl(fd, F_GETFD, 0);
+    if (flags == -1)
+        return -1;
+    return !(flags & FD_CLOEXEC);
+}
+    """],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+
+def set_inheritable(fd, inheritable):
+    error = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', error)
+
+def get_inheritable(fd):
+    res = c_get_inheritable(fd)
+    res = handle_posix_error('get_inheritable', res)
+    return res != 0
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -572,3 +572,12 @@
         os.close(dirfd)
     assert tmpdir.join('file').check(exists=False)
     assert tmpdir.join('file2').check(exists=True)
+
+def test_set_inheritable():
+    fd1, fd2 = os.pipe()
+    rposix.set_inheritable(fd1, True)
+    assert rposix.get_inheritable(fd1) == True
+    rposix.set_inheritable(fd1, False)
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)

From pypy.commits at gmail.com  Tue Aug 16 14:06:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 11:06:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Allow (and ignore) duplicate names in a
 single __slots__ specification.
Message-ID: <57b3562f.041f1c0a.770e5.ae58@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86232:146f531d6c86
Date: 2016-08-16 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/146f531d6c86/

Log:	Allow (and ignore) duplicate names in a single __slots__
	specification. Believe it or not, the stdlib datetime.py does that.

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1388,3 +1388,7 @@
         assert not self.compares_by_identity(X)
         del X.__eq__
         assert self.compares_by_identity(X)
+
+    def test_duplicate_slot_name(self):
+        class X:   # does not raise
+            __slots__ = 'a', 'a'
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1076,9 +1076,14 @@
     # create member
     slot_name = mangle(slot_name, w_self.name)
     if slot_name in w_self.dict_w:
-        raise oefmt(space.w_ValueError,
-                    "%R in __slots__ conflicts with class variable",
-                    w_slot_name)
+        w_prev = w_self.dict_w[slot_name]
+        if isinstance(w_prev, Member) and w_prev.w_cls is w_self:
+            pass   # special case: duplicate __slots__ entry, ignored
+                   # (e.g. occurs in datetime.py, fwiw)
+        else:
+            raise oefmt(space.w_ValueError,
+                        "%R in __slots__ conflicts with class variable",
+                        w_slot_name)
     else:
         # Force interning of slot names.
         slot_name = space.str_w(space.new_interned_str(slot_name))

From pypy.commits at gmail.com  Tue Aug 16 16:25:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 16 Aug 2016 13:25:22 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57b376b2.43681c0a.f6848.cc0f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r777:b516800af97c
Date: 2016-08-16 22:25 +0200
http://bitbucket.org/pypy/pypy.org/changeset/b516800af97c/

Log:	update the values

diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $30855 of $80000 (38.6%)
+   $30870 of $80000 (38.6%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $23511 left
+    we have $23524 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Tue Aug 16 18:44:46 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 16 Aug 2016 15:44:46 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: refactored changs from Aditi: all
 string tests now pass
Message-ID: <57b3975e.45c8c20a.dc13a.bf1b@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86234:8d97e46d038c
Date: 2016-08-16 15:40 -0700
http://bitbucket.org/pypy/pypy/changeset/8d97e46d038c/

Log:	refactored changs from Aditi: all string tests now pass

diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -158,14 +158,16 @@
     return _c_call_r(cppmethod, cppobject, nargs, args)
 _c_call_s = rffi.llexternal(
     "cppyy_call_s",
-    [C_METHOD, C_OBJECT, rffi.INT, rffi.VOIDP, rffi.INTP], rffi.CCHARP,
+    [C_METHOD, C_OBJECT, rffi.INT, rffi.VOIDP, rffi.SIZE_TP], rffi.CCHARP,
     releasegil=ts_call,
     compilation_info=backend.eci)
 def c_call_s(space, cppmethod, cppobject, nargs, args):
-    length = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-    cstr = _c_call_s(cppmethod, cppobject, nargs, args, length)
-    cstr_len = int(length[0])
-    lltype.free(length, flavor='raw')
+    length = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
+    try:
+        cstr = _c_call_s(cppmethod, cppobject, nargs, args, length)
+        cstr_len = int(length[0])
+    finally:
+        lltype.free(length, flavor='raw')
     return cstr, cstr_len
 
 _c_constructor = rffi.llexternal(
@@ -548,11 +550,11 @@
     [rffi.CCHARP, rffi.SIZE_T], C_OBJECT,
     releasegil=ts_helper,
     compilation_info=backend.eci)
-def c_charp2stdstring(space, svalue, sz):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2stdstring(charp, sz)
-    rffi.free_charp(charp)
-    return result
+def c_charp2stdstring(space, pystr, sz):
+    cstr = rffi.str2charp(pystr)
+    cppstr = _c_charp2stdstring(cstr, sz)
+    rffi.free_charp(cstr)
+    return cppstr
 _c_stdstring2stdstring = rffi.llexternal(
     "cppyy_stdstring2stdstring",
     [C_OBJECT], C_OBJECT,
diff --git a/pypy/module/cppyy/capi/cling_capi.py b/pypy/module/cppyy/capi/cling_capi.py
--- a/pypy/module/cppyy/capi/cling_capi.py
+++ b/pypy/module/cppyy/capi/cling_capi.py
@@ -1,9 +1,13 @@
 import py, os
 
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import libffi, rdynload
 
+from pypy.module.cppyy.capi.capi_types import C_OBJECT
+
 __all__ = ['identify', 'std_string_name', 'eci', 'c_load_dictionary']
 
 pkgpath = py.path.local(__file__).dirpath().join(os.pardir)
@@ -68,11 +72,42 @@
     pch = _c_load_dictionary(name)
     return pch
 
+_c_stdstring2charp = rffi.llexternal(
+    "cppyy_stdstring2charp",
+    [C_OBJECT, rffi.SIZE_TP], rffi.CCHARP,
+    releasegil=ts_helper,
+    compilation_info=eci)
+def c_stdstring2charp(space, cppstr):
+    sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
+    try:
+        cstr = _c_stdstring2charp(cppstr, sz)
+        cstr_len = int(sz[0])
+    finally:
+        lltype.free(sz, flavor='raw')
+    return rffi.charpsize2str(cstr, cstr_len)
 
-# Cling-specific pythonizations
+# pythonizations
+def stdstring_c_str(space, w_self):
+    """Return a python string taking into account \0"""
+
+    from pypy.module.cppyy import interp_cppyy
+    cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    return space.wrap(c_stdstring2charp(space, cppstr._rawobject))
+
+# setup pythonizations for later use at run-time
+_pythonizations = {}
 def register_pythonizations(space):
     "NOT_RPYTHON"
-    pass
+
+    allfuncs = [
+
+        ### std::string
+        stdstring_c_str,
+
+    ]
+
+    for f in allfuncs:
+        _pythonizations[f.__name__] = space.wrap(interp2app(f))
 
 def _method_alias(space, w_pycppclass, m1, m2):
     space.setattr(w_pycppclass, space.wrap(m1),
@@ -80,4 +115,6 @@
 
 def pythonize(space, name, w_pycppclass):
     if name == "string":
+        space.setattr(w_pycppclass, space.wrap("c_str"), _pythonizations["stdstring_c_str"])
         _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
+        _method_alias(space, w_pycppclass, "__str__",           "c_str")
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -59,7 +59,7 @@
     RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
     RPY_EXTERN
-    char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, int* length);
+    char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length);
 
     RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
@@ -179,6 +179,8 @@
     RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str, size_t sz);
     RPY_EXTERN
+    char* cppyy_stdstring2charp(cppyy_object_t ptr, size_t* lsz);
+    RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/include/cpp_cppyy.h b/pypy/module/cppyy/include/cpp_cppyy.h
--- a/pypy/module/cppyy/include/cpp_cppyy.h
+++ b/pypy/module/cppyy/include/cpp_cppyy.h
@@ -62,7 +62,7 @@
    Double_t     CallD( TCppMethod_t method, TCppObject_t self, void* args );
    LongDouble_t CallLD( TCppMethod_t method, TCppObject_t self, void* args );
    void*        CallR( TCppMethod_t method, TCppObject_t self, void* args );
-   Char_t*      CallS( TCppMethod_t method, TCppObject_t self, void* args, int* length );
+   Char_t*      CallS( TCppMethod_t method, TCppObject_t self, void* args, size_t* length );
    TCppObject_t CallConstructor( TCppMethod_t method, TCppType_t type, void* args );
    void         CallDestructor( TCppType_t type, TCppObject_t self );
    TCppObject_t CallO( TCppMethod_t method, TCppObject_t self, void* args, TCppType_t result_type );
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -479,7 +479,7 @@
 }
 
 Char_t* Cppyy::CallS(
-      TCppMethod_t method, TCppObject_t self, void* args, int* length )
+      TCppMethod_t method, TCppObject_t self, void* args, size_t* length )
 {
    char* cstr = nullptr;
    TClassRef cr("std::string");
@@ -1202,9 +1202,9 @@
 }
 
 char* cppyy_call_s(
-        cppyy_method_t method, cppyy_object_t self, int nargs, void* args, int* length) {
+        cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* lsz) {
     std::vector<TParameter> parvec = vsargs_to_parvec(args, nargs);
-    return Cppyy::CallS(method, (void*)self, &parvec, length);
+    return Cppyy::CallS(method, (void*)self, &parvec, lsz);
 }
 
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args) {
@@ -1463,10 +1463,15 @@
     free(ptr);
 }
 
-cppyy_object_t cppyy_charp2stdstring(const char* str, size_t sz){
+cppyy_object_t cppyy_charp2stdstring(const char* str, size_t sz) {
     return (cppyy_object_t)new std::string(str, sz);
 }
 
+char* cppyy_stdstring2charp(cppyy_object_t ptr, size_t* lsz) {
+    *lsz = ((std::string*)ptr)->size();
+    return (char*)((std::string*)ptr)->data();
+}
+
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr){
     return (cppyy_object_t)new std::string(*(std::string*)ptr);
 }

From pypy.commits at gmail.com  Tue Aug 16 19:01:09 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 16 Aug 2016 16:01:09 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: from Aditi: loadable_capi fixes
Message-ID: <57b39b35.a699c20a.af953.2fbc@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86235:373896495865
Date: 2016-08-16 15:56 -0700
http://bitbucket.org/pypy/pypy/changeset/373896495865/

Log:	from Aditi: loadable_capi fixes

diff --git a/pypy/module/cppyy/capi/cling_capi.py b/pypy/module/cppyy/capi/cling_capi.py
--- a/pypy/module/cppyy/capi/cling_capi.py
+++ b/pypy/module/cppyy/capi/cling_capi.py
@@ -86,6 +86,7 @@
         lltype.free(sz, flavor='raw')
     return rffi.charpsize2str(cstr, cstr_len)
 
+# TODO: factor these out ...
 # pythonizations
 def stdstring_c_str(space, w_self):
     """Return a python string taking into account \0"""
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -146,7 +146,7 @@
             'call_d'       : ([c_method, c_object, c_int, c_voidp],   c_double),
 
             'call_r'       : ([c_method, c_object, c_int, c_voidp],   c_voidp),
-            # call_s actually takes an intp as last parameter, but this will do
+            # call_s actually takes an size_t* as last parameter, but this will do
             'call_s'       : ([c_method, c_object, c_int, c_voidp, c_voidp],    c_ccharp),
 
             'constructor'  : ([c_method, c_object, c_int, c_voidp],   c_object),
@@ -337,8 +337,8 @@
     args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
     return _cdata_to_ptr(space, call_capi(space, 'call_r', args))
 def c_call_s(space, cppmethod, cppobject, nargs, cargs):
-    length = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs), _Args(vp=length)]
+    length = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
+    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs), _Arg(vp=length)]
     cstr = call_capi(space, 'call_s', args)
     cstr_len = int(length[0])
     lltype.free(length, flavor='raw')
@@ -373,7 +373,7 @@
     return space.bool_w(call_capi(space, 'is_namespace', [_Arg(h=scope)]))
 def c_is_template(space, name):
     return space.bool_w(call_capi(space, 'is_template', [_Arg(s=name)]))
-def c_is_abstract(space, scope):
+def c_is_abstract(space, cpptype):
     return space.bool_w(call_capi(space, 'is_abstract', [_Arg(h=cpptype)]))
 def c_is_enum(space, name):
     return space.bool_w(call_capi(space, 'is_enum', [_Arg(s=name)]))
@@ -525,13 +525,53 @@
 def c_charp2stdstring(space, svalue, sz):
     return _cdata_to_cobject(
         space, call_capi(space, 'charp2stdstring', [_Arg(s=svalue), _Arg(l=sz)]))
+_c_stdstring2charp = rffi.llexternal(
+    "cppyy_stdstring2charp",
+    [C_OBJECT, rffi.SIZE_TP], rffi.CCHARP,
+    releasegil=ts_helper,
+    compilation_info=eci)
+def c_stdstring2charp(space, cppstr):
+    sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
+    try:
+        cstr = call_capi(space, 'stdstring2charp', [_Arg(h=cppstr), _Arg(vp=sz)])
+        cstr_len = int(sz[0])
+    finally:
+        lltype.free(sz, flavor='raw')
+    return rffi.charpsize2str(cstr, cstr_len)
 def c_stdstring2stdstring(space, cppobject):
     return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_Arg(h=cppobject)]))
 
-# loadable-capi-specific pythonizations (none, as the capi isn't known until runtime)
+
+# TODO: factor these out ...
+# pythonizations
+def stdstring_c_str(space, w_self):
+    """Return a python string taking into account \0"""
+
+    from pypy.module.cppyy import interp_cppyy
+    cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    return space.wrap(c_stdstring2charp(space, cppstr._rawobject))
+
+# setup pythonizations for later use at run-time
+_pythonizations = {}
 def register_pythonizations(space):
     "NOT_RPYTHON"
-    pass
+
+    allfuncs = [
+
+        ### std::string
+        stdstring_c_str,
+
+    ]
+
+    for f in allfuncs:
+        _pythonizations[f.__name__] = space.wrap(interp2app(f))
+
+def _method_alias(space, w_pycppclass, m1, m2):
+    space.setattr(w_pycppclass, space.wrap(m1),
+                  space.getattr(w_pycppclass, space.wrap(m2)))
 
 def pythonize(space, name, w_pycppclass):
-    pass
+    if name == "string":
+        space.setattr(w_pycppclass, space.wrap("c_str"), _pythonizations["stdstring_c_str"])
+        _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
+        _method_alias(space, w_pycppclass, "__str__",           "c_str")

From pypy.commits at gmail.com  Tue Aug 16 19:46:23 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 16 Aug 2016 16:46:23 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: cleanup
Message-ID: <57b3a5cf.151a1c0a.a5160.0c43@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86236:4333bf213e97
Date: 2016-08-16 16:41 -0700
http://bitbucket.org/pypy/pypy/changeset/4333bf213e97/

Log:	cleanup

diff --git a/pypy/module/cppyy/capi/cling_capi.py b/pypy/module/cppyy/capi/cling_capi.py
--- a/pypy/module/cppyy/capi/cling_capi.py
+++ b/pypy/module/cppyy/capi/cling_capi.py
@@ -1,6 +1,6 @@
 import py, os
 
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app
 
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.lltypesystem import rffi, lltype
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -3,6 +3,7 @@
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.tool import leakfinder
 
+from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.error import oefmt
 
 from pypy.module._cffi_backend import ctypefunc, ctypeprim, cdataobj, misc
@@ -218,6 +219,8 @@
             'free'                     : ([c_voidp],                  c_void),
 
             'charp2stdstring'          : ([c_ccharp, c_size_t],       c_object),
+            #stdstring2charp  actually takes an size_t* as last parameter, but this will do
+            'stdstring2charp'          : ([c_ccharp, c_voidp],        c_ccharp),
             'stdstring2stdstring'      : ([c_object],                 c_object),
         }
 
@@ -525,11 +528,6 @@
 def c_charp2stdstring(space, svalue, sz):
     return _cdata_to_cobject(
         space, call_capi(space, 'charp2stdstring', [_Arg(s=svalue), _Arg(l=sz)]))
-_c_stdstring2charp = rffi.llexternal(
-    "cppyy_stdstring2charp",
-    [C_OBJECT, rffi.SIZE_TP], rffi.CCHARP,
-    releasegil=ts_helper,
-    compilation_info=eci)
 def c_stdstring2charp(space, cppstr):
     sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
     try:
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -179,7 +179,7 @@
     RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str, size_t sz);
     RPY_EXTERN
-    char* cppyy_stdstring2charp(cppyy_object_t ptr, size_t* lsz);
+    const char* cppyy_stdstring2charp(cppyy_object_t ptr, size_t* lsz);
     RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -1467,9 +1467,9 @@
     return (cppyy_object_t)new std::string(str, sz);
 }
 
-char* cppyy_stdstring2charp(cppyy_object_t ptr, size_t* lsz) {
+const char* cppyy_stdstring2charp(cppyy_object_t ptr, size_t* lsz) {
     *lsz = ((std::string*)ptr)->size();
-    return (char*)((std::string*)ptr)->data();
+    return ((std::string*)ptr)->data();
 }
 
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr){

From pypy.commits at gmail.com  Tue Aug 16 19:46:25 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 16 Aug 2016 16:46:25 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: raise same type exception if all
 overloads return that type; otherwise TypeError
Message-ID: <57b3a5d1.05371c0a.db028.ebc7@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86237:b71f0b85c644
Date: 2016-08-16 16:41 -0700
http://bitbucket.org/pypy/pypy/changeset/b71f0b85c644/

Log:	raise same type exception if all overloads return that type;
	otherwise TypeError

diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -544,6 +544,8 @@
         errmsg = 'none of the %d overloaded methods succeeded. Full details:' % len(self.functions)
         if hasattr(self.space, "fake"):     # FakeSpace fails errorstr (see below)
             raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
+        w_exc_type = None
+        all_same_type = True
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
@@ -552,6 +554,10 @@
                 # special case if there's just one function, to prevent clogging the error message
                 if len(self.functions) == 1:
                     raise
+                if w_exc_type is None:
+                    w_exc_type = e.w_type
+                elif all_same_type and not e.match(self.space, w_exc_type):
+                    all_same_type = False
                 errmsg += '\n  '+cppyyfunc.signature()+' =>\n'
                 errmsg += '    '+e.errorstr(self.space)
             except Exception as e:
@@ -560,7 +566,10 @@
                 errmsg += '\n  '+cppyyfunc.signature()+' =>\n'
                 errmsg += '    Exception: '+str(e)
 
-        raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
+        if all_same_type and w_exc_type is not None:
+            raise OperationError(w_exc_type, self.space.wrap(errmsg))
+        else:
+            raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
 
     def signature(self):
         sig = self.functions[0].signature()

From pypy.commits at gmail.com  Tue Aug 16 20:05:32 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 16 Aug 2016 17:05:32 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: link examples with -lCore for the
 benefit of loadable_capi.py
Message-ID: <57b3aa4c.d41a1c0a.68336.9791@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86238:6c2423d9357d
Date: 2016-08-16 17:01 -0700
http://bitbucket.org/pypy/pypy/changeset/6c2423d9357d/

Log:	link examples with -lCore for the benefit of loadable_capi.py

diff --git a/pypy/module/cppyy/test/Makefile b/pypy/module/cppyy/test/Makefile
--- a/pypy/module/cppyy/test/Makefile
+++ b/pypy/module/cppyy/test/Makefile
@@ -19,7 +19,7 @@
     cppflags=-pthread -std=c++11 -m64 -I./include -L./lib64 -L./lib
   else
     genreflex=$(ROOTSYS)/bin/genreflex
-    cppflags=$(shell $(ROOTSYS)/bin/root-config --cflags) $(shell $(ROOTSYS)/bin/root-config --ldflags)
+    cppflags=$(shell $(ROOTSYS)/bin/root-config --cflags) $(shell $(ROOTSYS)/bin/root-config --ldflags) -L$(shell $(ROOTSYS)/bin/root-config --libdir) -lCore
   endif
 endif
 

From pypy.commits at gmail.com  Wed Aug 17 03:29:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 00:29:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: missing import
Message-ID: <57b41251.87941c0a.91280.78a2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86239:7b764c792a9f
Date: 2016-08-17 09:28 +0200
http://bitbucket.org/pypy/pypy/changeset/7b764c792a9f/

Log:	missing import

diff --git a/lib-python/3/test/test_class.py b/lib-python/3/test/test_class.py
--- a/lib-python/3/test/test_class.py
+++ b/lib-python/3/test/test_class.py
@@ -2,6 +2,7 @@
 
 import sys
 import unittest
+from test import support
 
 
 testmeths = [

From pypy.commits at gmail.com  Wed Aug 17 05:06:14 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 17 Aug 2016 02:06:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: add a fake jitdriver to the jitdriver
 static data (fixes many tests in test_ajit)
Message-ID: <57b42906.44ce1c0a.99144.97cb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r86241:0b56b66d816b
Date: 2016-08-17 11:04 +0200
http://bitbucket.org/pypy/pypy/changeset/0b56b66d816b/

Log:	add a fake jitdriver to the jitdriver static data (fixes many tests
	in test_ajit)

diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -64,6 +64,10 @@
     testself.all_graphs = graphs
     result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0]
 
+
+    class FakeJitDriver:
+        name = 'fakejitdriver'
+
     class FakeJitDriverSD:
         num_green_args = 0
         portal_graph = graphs[0]
@@ -72,6 +76,7 @@
         result_type = result_kind
         portal_runner_ptr = "???"
         vec = False
+        jitdriver = FakeJitDriver()
 
     stats = history.Stats(None)
     cpu = CPUClass(rtyper, stats, None, False)

From pypy.commits at gmail.com  Wed Aug 17 05:06:12 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 17 Aug 2016 02:06:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: merge py3.5
Message-ID: <57b42904.919a1c0a.60c6a.942c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86240:d8e14af5ffed
Date: 2016-08-17 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/d8e14af5ffed/

Log:	merge py3.5

diff too long, truncating to 2000 out of 4486 lines

diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib-python/3/code.py b/lib-python/3/code.py
--- a/lib-python/3/code.py
+++ b/lib-python/3/code.py
@@ -140,32 +140,15 @@
         sys.last_type, sys.last_value, last_tb = ei = sys.exc_info()
         sys.last_traceback = last_tb
         try:
-            lines = []
-            for value, tb in traceback._iter_chain(*ei[1:]):
-                if isinstance(value, str):
-                    lines.append(value)
-                    lines.append('\n')
-                    continue
-                if tb:
-                    tblist = traceback.extract_tb(tb)
-                    if tb is last_tb:
-                        # The last traceback includes the frame we
-                        # exec'd in
-                        del tblist[:1]
-                    tblines = traceback.format_list(tblist)
-                    if tblines:
-                        lines.append("Traceback (most recent call last):\n")
-                        lines.extend(tblines)
-                lines.extend(traceback.format_exception_only(type(value),
-                                                             value))
+            lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next)
+            if sys.excepthook is sys.__excepthook__:
+                self.write(''.join(lines))
+            else:
+                # If someone has set sys.excepthook, we let that take precedence
+                # over self.write
+                sys.excepthook(ei[0], ei[1], last_tb)
         finally:
-            tblist = last_tb = ei = None
-        if sys.excepthook is sys.__excepthook__:
-            self.write(''.join(lines))
-        else:
-            # If someone has set sys.excepthook, we let that take precedence
-            # over self.write
-            sys.excepthook(sys.last_type, sys.last_value, last_tb)
+            last_tb = ei = None
 
     def write(self, data):
         """Write a string.
diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py
--- a/lib-python/3/distutils/command/build_ext.py
+++ b/lib-python/3/distutils/command/build_ext.py
@@ -11,7 +11,6 @@
 from distutils.core import Command
 from distutils.errors import *
 from distutils.sysconfig import customize_compiler, get_python_version
-from distutils.sysconfig import get_config_h_filename
 from distutils.dep_util import newer_group
 from distutils.extension import Extension
 from distutils.util import get_platform
@@ -30,6 +29,7 @@
     show_compilers()
 
 def _get_c_extension_suffix():
+    import importlib
     suffixes = importlib.machinery.EXTENSION_SUFFIXES
     return suffixes[0] if suffixes else None
 
@@ -204,6 +204,7 @@
             # this allows distutils on windows to work in the source tree
             if 0:
                 # pypy has no config_h_filename directory
+                from distutils.sysconfig import get_config_h_filename
                 self.include_dirs.append(os.path.dirname(get_config_h_filename()))
             _sys_home = getattr(sys, '_home', None)
             if _sys_home:
diff --git a/lib-python/3/site.py b/lib-python/3/site.py
--- a/lib-python/3/site.py
+++ b/lib-python/3/site.py
@@ -378,8 +378,8 @@
         license = "See https://www.python.org/psf/license/"
         licenseargs = (license, files, dirs)
 
-    builtins.credits = _Printer("credits", credits)
-    builtins.license = _Printer("license", *licenseargs)
+    builtins.credits = _sitebuiltins._Printer("credits", credits)
+    builtins.license = _sitebuiltins._Printer("license", *licenseargs)
 
 def sethelper():
     builtins.help = _sitebuiltins._Helper()
diff --git a/lib-python/3/test/test_class.py b/lib-python/3/test/test_class.py
--- a/lib-python/3/test/test_class.py
+++ b/lib-python/3/test/test_class.py
@@ -2,6 +2,7 @@
 
 import sys
 import unittest
+from test import support
 
 
 testmeths = [
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -105,3 +105,26 @@
 .. branch: ep2016sprint
 
 Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -114,8 +114,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1322,6 +1322,25 @@
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
 
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts
+
     def test_call_function_var(self):
         source = """call(*me)"""
         code, blocks = generate_function_code(source, self.space)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1375,10 +1375,11 @@
         for i in range(itemcount, 0, -1):
             w_item = self.peekvalue(i-1)
             # cannot use w_sum.update, w_item might not be a set
-            iterator = w_item.itervalues()
+            iterator = space.iter(w_item)
             while True:
-                w_value = iterator.next_value()
-                if w_value is None:
+                try:
+                    w_value = space.next(iterator)
+                except OperationError:
                     break
                 w_sum.add(w_value)
         while itemcount != 0:
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -209,7 +209,6 @@
         self.check(['-c', 'pass'], {'PYTHONNOUSERSITE': '1'}, sys_argv=['-c'],
                    run_command='pass', **expected)
 
-
 class TestInteraction:
     """
     These tests require pexpect (UNIX-only).
@@ -1152,4 +1151,3 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -134,7 +134,7 @@
         assert self.space.lookup(w_instance, "gobbledygook") is None
         w_instance = self.space.appexec([], """():
             class Lookup(object):
-                "bla" 
+                "bla"
             return Lookup()""")
         assert self.space.str_w(self.space.lookup(w_instance, "__doc__")) == "bla"
 
@@ -148,7 +148,7 @@
         assert is_callable(w_func)
         w_lambda_func = self.space.appexec([], "(): return lambda: True")
         assert is_callable(w_lambda_func)
-        
+
         w_instance = self.space.appexec([], """():
             class Call(object):
                 def __call__(self): pass
@@ -308,7 +308,7 @@
 
     def test_call_obj_args(self):
         from pypy.interpreter.argument import Arguments
-        
+
         space = self.space
 
         w_f = space.appexec([], """():
@@ -333,7 +333,7 @@
         assert w_x is w_9
         assert w_y is w_1
 
-        w_res = space.call_obj_args(w_a, w_9, Arguments(space, []))        
+        w_res = space.call_obj_args(w_a, w_9, Arguments(space, []))
         assert w_res is w_9
 
     def test_compare_by_iteration(self):
@@ -383,7 +383,7 @@
         assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('g')))
         assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('h')))
 
-class TestModuleMinimal: 
+class TestModuleMinimal:
     def test_sys_exists(self):
         assert self.space.sys
 
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -18,3 +18,31 @@
         loop.run_until_complete(f())
         print("done with async loop")
         """
+    
+    def test_asynchronous_context_managers(self):
+        """
+import encodings.idna
+import asyncio
+
+class Corotest(object):
+    def __init__(self):
+        self.res = "-"
+    
+    async def coro(self, name, lock):
+        self.res += ' coro {}: waiting for lock -'.format(name)
+        async with lock:
+            self.res += ' coro {}: holding the lock -'.format(name)
+            await asyncio.sleep(1)
+            self.res += ' coro {}: releasing the lock -'.format(name)
+
+cor = Corotest()
+loop = asyncio.get_event_loop()
+lock = asyncio.Lock()
+coros = asyncio.gather(cor.coro(1, lock), cor.coro(2, lock))
+try:
+    loop.run_until_complete(coros)
+finally:
+    loop.close()
+
+assert cor.res == "- coro 1: waiting for lock - coro 1: holding the lock - coro 2: waiting for lock - coro 1: releasing the lock - coro 2: holding the lock - coro 2: releasing the lock -"
+        """
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.7.0"
+VERSION = "1.8.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -157,11 +157,13 @@
         mustfree_max_plus_1 = 0
         buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
         try:
+            keepalives = [None] * len(args_w)    # None or strings
             for i in range(len(args_w)):
                 data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                 w_obj = args_w[i]
                 argtype = self.fargs[i]
-                if argtype.convert_argument_from_object(data, w_obj):
+                if argtype.convert_argument_from_object(data, w_obj,
+                                                        keepalives, i):
                     # argtype is a pointer type, and w_obj a list/tuple/str
                     mustfree_max_plus_1 = i + 1
 
@@ -177,9 +179,13 @@
                 if isinstance(argtype, W_CTypePointer):
                     data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                     flag = get_mustfree_flag(data)
+                    raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                     if flag == 1:
-                        raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
+                    elif flag >= 4:
+                        value = keepalives[i]
+                        assert value is not None
+                        rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
             lltype.free(buffer, flavor='raw')
             keepalive_until_here(args_w)
         return w_res
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -83,7 +83,7 @@
         raise oefmt(space.w_TypeError, "cannot initialize cdata '%s'",
                     self.name)
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         self.convert_from_object(cdata, w_ob)
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -16,8 +16,8 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
+    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
     length = -1
 
     def __init__(self, space, size, extra, extra_position, ctitem,
@@ -30,6 +30,9 @@
         #  - for functions, it is the return type
         self.ctitem = ctitem
         self.can_cast_anything = could_cast_anything and ctitem.cast_anything
+        self.accept_str = (self.can_cast_anything or
+                            (ctitem.is_primitive_integer and
+                             ctitem.size == rffi.sizeof(lltype.Char)))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -72,9 +75,7 @@
                 pass
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
-        elif (self.can_cast_anything or
-              (self.ctitem.is_primitive_integer and
-               self.ctitem.size == rffi.sizeof(lltype.Char))):
+        elif self.accept_str:
             if not space.isinstance_w(w_ob, space.w_str):
                 raise self._convert_error("bytes or list or tuple", w_ob)
             s = space.str_w(w_ob)
@@ -262,8 +263,16 @@
         else:
             return lltype.nullptr(rffi.CCHARP.TO)
 
-    def _prepare_pointer_call_argument(self, w_init, cdata):
+    def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
         space = self.space
+        if self.accept_str and space.isinstance_w(w_init, space.w_str):
+            # special case to optimize strings passed to a "char *" argument
+            value = space.bytes_w(w_init)
+            keepalives[i] = value
+            buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
+            rffi.cast(rffi.CCHARPP, cdata)[0] = buf
+            return ord(buf_flag)    # 4, 5 or 6
+        #
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
@@ -300,10 +309,11 @@
         rffi.cast(rffi.CCHARPP, cdata)[0] = result
         return 1
 
-    def convert_argument_from_object(self, cdata, w_ob):
+    def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
         result = (not isinstance(w_ob, cdataobj.W_CData) and
-                  self._prepare_pointer_call_argument(w_ob, cdata))
+                  self._prepare_pointer_call_argument(w_ob, cdata,
+                                                      keepalives, i))
         if result == 0:
             self.convert_from_object(cdata, w_ob)
         set_mustfree_flag(cdata, result)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -353,7 +353,7 @@
 'array.array' or numpy arrays."""
         #
         w_ctchara = newtype._new_chara_type(self.space)
-        return func.from_buffer(self.space, w_ctchara, w_python_buffer)
+        return func._from_buffer(self.space, w_ctchara, w_python_buffer)
 
 
     @unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated
+from rpython.rlib import jit
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -132,17 +133,66 @@
         raise oefmt(space.w_TypeError,
                     "needs 'char[]', got '%s'", w_ctype.name)
     #
+    return _from_buffer(space, w_ctype, w_x)
+
+def _from_buffer(space, w_ctype, w_x):
     buf = _fetch_as_read_buffer(space, w_x)
-    try:
-        _cdata = buf.get_raw_address()
-    except ValueError:
-        raise oefmt(space.w_TypeError,
-                    "from_buffer() got a '%T' object, which supports the "
-                    "buffer interface but cannot be rendered as a plain "
-                    "raw address on PyPy", w_x)
+    if space.isinstance_w(w_x, space.w_str):
+        _cdata = get_raw_address_of_string(space, w_x)
+    else:
+        try:
+            _cdata = buf.get_raw_address()
+        except ValueError:
+            raise oefmt(space.w_TypeError,
+                        "from_buffer() got a '%T' object, which supports the "
+                        "buffer interface but cannot be rendered as a plain "
+                        "raw address on PyPy", w_x)
     #
     return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
 
+# ____________________________________________________________
+
+class RawBytes(object):
+    def __init__(self, string):
+        self.ptr = rffi.str2charp(string, track_allocation=False)
+    def __del__(self):
+        rffi.free_charp(self.ptr, track_allocation=False)
+
+class RawBytesCache(object):
+    def __init__(self, space):
+        from pypy.interpreter.baseobjspace import W_Root
+        from rpython.rlib import rweakref
+        self.wdict = rweakref.RWeakKeyDictionary(W_Root, RawBytes)
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(space, w_x):
+    """Special case for ffi.from_buffer(string).  Returns a 'char *' that
+    is valid as long as the string object is alive.  Two calls to
+    ffi.from_buffer(same_string) are guaranteed to return the same pointer.
+    """
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rtyper.lltypesystem.rstr import STR
+    from rpython.rtyper.lltypesystem import llmemory
+    from rpython.rlib import rgc
+
+    cache = space.fromcache(RawBytesCache)
+    rawbytes = cache.wdict.get(w_x)
+    if rawbytes is None:
+        data = space.str_w(w_x)
+        if we_are_translated() and not rgc.can_move(data):
+            lldata = llstr(data)
+            data_start = (llmemory.cast_ptr_to_adr(lldata) +
+                          rffi.offsetof(STR, 'chars') +
+                          llmemory.itemoffsetof(STR.chars, 0))
+            data_start = rffi.cast(rffi.CCHARP, data_start)
+            data_start[len(data)] = '\x00'   # write the final extra null
+            return data_start
+        rawbytes = RawBytes(data)
+        cache.wdict.set(w_x, rawbytes)
+    return rawbytes.ptr
+
+# ____________________________________________________________
+
 
 def unsafe_escaping_ptr_for_ptr_or_array(w_cdata):
     if not w_cdata.ctype.is_nonfunc_pointer_or_array:
diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py
--- a/pypy/module/_cffi_backend/parse_c_type.py
+++ b/pypy/module/_cffi_backend/parse_c_type.py
@@ -97,11 +97,8 @@
                                   [rffi.INT], rffi.CCHARP)
 
 def parse_c_type(info, input):
-    p_input = rffi.str2charp(input)
-    try:
+    with rffi.scoped_view_charp(input) as p_input:
         res = ll_parse_c_type(info, p_input)
-    finally:
-        rffi.free_charp(p_input)
     return rffi.cast(lltype.Signed, res)
 
 NULL_CTX = lltype.nullptr(PCTX.TO)
@@ -130,15 +127,13 @@
     return rffi.getintfield(src_ctx, 'c_num_types')
 
 def search_in_globals(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_globals(ctx, c_name,
-                                  rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_globals(ctx, c_name,
+                                      rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
 
 def search_in_struct_unions(ctx, name):
-    c_name = rffi.str2charp(name)
-    result = ll_search_in_struct_unions(ctx, c_name,
-                                        rffi.cast(rffi.SIZE_T, len(name)))
-    rffi.free_charp(c_name)
+    with rffi.scoped_view_charp(name) as c_name:
+        result = ll_search_in_struct_unions(ctx, c_name,
+                                            rffi.cast(rffi.SIZE_T, len(name)))
     return rffi.cast(lltype.Signed, result)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3330,13 +3330,18 @@
     BChar = new_primitive_type("char")
     BCharP = new_pointer_type(BChar)
     BCharA = new_array_type(BCharP, None)
-    py.test.raises(TypeError, from_buffer, BCharA, b"foo")
+    p1 = from_buffer(BCharA, b"foo")
+    assert p1 == from_buffer(BCharA, b"foo")
+    import gc; gc.collect()
+    assert p1 == from_buffer(BCharA, b"foo")
     py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
     try:
         from __builtin__ import buffer
     except ImportError:
         pass
     else:
+        # from_buffer(buffer(b"foo")) does not work, because it's not
+        # implemented on pypy; only from_buffer(b"foo") works.
         py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
         py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
     try:
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -402,21 +402,20 @@
             _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
         from rpython.rlib import rwin32
 
-        charp = rffi.str2charp(buf)
-        written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                    flavor='raw')
-        try:
-            result = _WriteFile(
-                self.handle, rffi.ptradd(charp, offset),
-                size, written_ptr, rffi.NULL)
+        with rffi.scoped_view_charp(buf) as charp:
+            written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
+                                        flavor='raw')
+            try:
+                result = _WriteFile(
+                    self.handle, rffi.ptradd(charp, offset),
+                    size, written_ptr, rffi.NULL)
 
-            if (result == 0 and
-                rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
-                raise oefmt(space.w_ValueError,
-                            "Cannot send %d bytes over connection", size)
-        finally:
-            rffi.free_charp(charp)
-            lltype.free(written_ptr, flavor='raw')
+                if (result == 0 and
+                    rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
+                    raise oefmt(space.w_ValueError,
+                                "Cannot send %d bytes over connection", size)
+            finally:
+                lltype.free(written_ptr, flavor='raw')
 
     def do_recv_string(self, space, buflength, maxlength):
         from pypy.module._multiprocessing.interp_win32 import (
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -15,8 +15,9 @@
 
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
-        includes=['unistd.h', 'sys/syscall.h'])
+        includes=['unistd.h', 'sys/syscall.h', 'sys/stat.h'])
     HAVE_SYS_SYSCALL_H = platform.Has("syscall")
+    HAVE_SYS_STAT_H = platform.Has("stat")
     HAVE_SETSID = platform.Has("setsid")
 
 config = platform.configure(CConfig)
@@ -29,6 +30,8 @@
 compile_extra = []
 if config['HAVE_SYS_SYSCALL_H']:
     compile_extra.append("-DHAVE_SYS_SYSCALL_H")
+if config['HAVE_SYS_STAT_H']:
+    compile_extra.append("-DHAVE_SYS_STAT_H")
 if config['HAVE_SETSID']:
     compile_extra.append("-DHAVE_SETSID")
 
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,6 +1,7 @@
 import sys, os
-import py
+import pytest
 from pypy.tool.pytest.objspace import gettestobjspace
+from pypy.interpreter.gateway import interp2app
 from rpython.tool.udir import udir
 from rpython.rlib import rsocket
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -13,8 +14,6 @@
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
     mod.path = udir.join('fd')
     mod.path.write('fo')
-    mod.raises = py.test.raises # make raises available from app-level tests
-    mod.skip = py.test.skip
 
 def test_gethostname():
     host = space.appexec([w_socket], "(_socket): return _socket.gethostname()")
@@ -42,7 +41,7 @@
     for host in ["localhost", "127.0.0.1", "::1"]:
         if host == "::1" and not ipv6:
             from pypy.interpreter.error import OperationError
-            with py.test.raises(OperationError):
+            with pytest.raises(OperationError):
                 space.appexec([w_socket, space.wrap(host)],
                               "(_socket, host): return _socket.gethostbyaddr(host)")
             continue
@@ -58,14 +57,14 @@
     assert space.unwrap(port) == 25
     # 1 arg version
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyname second argument is not optional before python 2.4")
+        pytest.skip("getservbyname second argument is not optional before python 2.4")
     port = space.appexec([w_socket, space.wrap(name)],
                         "(_socket, name): return _socket.getservbyname(name)")
     assert space.unwrap(port) == 25
 
 def test_getservbyport():
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyport does not exist before python 2.4")
+        pytest.skip("getservbyport does not exist before python 2.4")
     port = 25
     # 2 args version
     name = space.appexec([w_socket, space.wrap(port)],
@@ -139,7 +138,7 @@
 
 def test_pton_ntop_ipv4():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     tests = [
         ("123.45.67.89", "\x7b\x2d\x43\x59"),
         ("0.0.0.0", "\x00" * 4),
@@ -155,9 +154,9 @@
 
 def test_ntop_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -176,9 +175,9 @@
 
 def test_pton_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -197,7 +196,7 @@
         assert space.unwrap(w_packed) == packed
 
 def test_has_ipv6():
-    py.test.skip("has_ipv6 is always True on PyPy for now")
+    pytest.skip("has_ipv6 is always True on PyPy for now")
     res = space.appexec([w_socket], "(_socket): return _socket.has_ipv6")
     assert space.unwrap(res) == socket.has_ipv6
 
@@ -231,7 +230,7 @@
 def test_addr_raw_packet():
     from pypy.module._socket.interp_socket import addr_as_object
     if not hasattr(rsocket._c, 'sockaddr_ll'):
-        py.test.skip("posix specific test")
+        pytest.skip("posix specific test")
     # HACK: To get the correct interface number of lo, which in most cases is 1,
     # but can be anything (i.e. 39), we need to call the libc function
     # if_nametoindex to get the correct index
@@ -653,11 +652,11 @@
 class AppTestNetlink:
     def setup_class(cls):
         if not hasattr(os, 'getpid'):
-            py.test.skip("AF_NETLINK needs os.getpid()")
+            pytest.skip("AF_NETLINK needs os.getpid()")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_NETLINK')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_NETLINK on this platform")
+            pytest.skip("no AF_NETLINK on this platform")
         cls.space = space
 
     def test_connect_to_kernel_netlink_routing_socket(self):
@@ -673,11 +672,11 @@
 class AppTestPacket:
     def setup_class(cls):
         if not hasattr(os, 'getuid') or os.getuid() != 0:
-            py.test.skip("AF_PACKET needs to be root for testing")
+            pytest.skip("AF_PACKET needs to be root for testing")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_PACKET')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_PACKET on this platform")
+            pytest.skip("no AF_PACKET on this platform")
         cls.space = space
 
     def test_convert_between_tuple_and_sockaddr_ll(self):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -146,7 +146,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         # set both server and client callbacks, because the context
@@ -158,7 +158,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def advertiseNPN_cb(s, data_ptr, len_ptr, args):
@@ -192,7 +192,7 @@
 
     def __init__(self, ctx, protos):
         self.protos = protos
-        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
         ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
 
         with rffi.scoped_str2charp(protos) as protos_buf:
@@ -204,7 +204,7 @@
 
     def __del__(self):
         rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.pinned, self.is_raw)
+            self.protos, self.buf, self.bufflag)
 
     @staticmethod
     def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
@@ -239,7 +239,7 @@
 
         Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
         bound on the entropy contained in string."""
-        with rffi.scoped_str2charp(string) as buf:
+        with rffi.scoped_nonmovingbuffer(string) as buf:
             libssl_RAND_add(buf, len(string), entropy)
 
     def _RAND_bytes(space, n, pseudo):
diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -537,9 +537,8 @@
     releasegil=ts_helper,
     compilation_info=backend.eci)
 def c_charp2stdstring(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2stdstring(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2stdstring(charp)
     return result
 _c_stdstring2stdstring = rffi.llexternal(
     "cppyy_stdstring2stdstring",
diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -82,9 +82,8 @@
     releasegil=ts_helper,
     compilation_info=eci)
 def c_charp2TString(space, svalue):
-    charp = rffi.str2charp(svalue)
-    result = _c_charp2TString(charp)
-    rffi.free_charp(charp)
+    with rffi.scoped_view_charp(svalue) as charp:
+        result = _c_charp2TString(charp)
     return result
 _c_TString2TString = rffi.llexternal(
     "cppyy_TString2TString",
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -65,6 +65,7 @@
                 else:    # only other use is sring
                     n = len(obj._string)
                     assert raw_string == rffi.cast(rffi.CCHARP, 0)
+                    # XXX could use rffi.get_nonmovingbuffer_final_null()
                     raw_string = rffi.str2charp(obj._string)
                     data = rffi.cast(rffi.CCHARPP, data)
                     data[0] = raw_string
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,6 +11,9 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+        c_fdopen, c_fileno,
+        c_fopen)# for tests
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import NameManager
@@ -84,44 +87,32 @@
 assert CONST_WSTRING == rffi.CWCHARP
 
 # FILE* interface
-FILEP = rffi.COpaquePtr('FILE')
 
 if sys.platform == 'win32':
     dash = '_'
 else:
     dash = ''
-fileno = rffi.llexternal(dash + 'fileno', [FILEP], rffi.INT)
-fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
-fdopen = rffi.llexternal(dash + 'fdopen', [rffi.INT, CONST_STRING],
-                  FILEP, save_err=rffi.RFFI_SAVE_ERRNO)
 
-_fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
-    if not is_valid_fd(fileno(fp)):
+    if not is_valid_fd(c_fileno(fp)):
         return -1
-    return _fclose(fp)
+    return c_fclose(fp)
 
-_fwrite = rffi.llexternal('fwrite',
-                         [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                         rffi.SIZE_T)
 def fwrite(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fwrite(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fwrite(buf, sz, n, fp)
 
-_fread = rffi.llexternal('fread',
-                        [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                        rffi.SIZE_T)
 def fread(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fread(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fread(buf, sz, n, fp)
 
 _feof = rffi.llexternal('feof', [FILEP], rffi.INT)
 def feof(fp):
-    validate_fd(fileno(fp))
+    validate_fd(c_fileno(fp))
     return _feof(fp)
 
 def is_valid_fp(fp):
-    return is_valid_fd(fileno(fp))
+    return is_valid_fd(c_fileno(fp))
 
 pypy_decl = 'pypy_decl.h'
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -96,7 +96,8 @@
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
             py_str.c_ob_size, len(s))
-    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    with rffi.scoped_nonmovingbuffer(s) as s_ptr:
+        rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
     py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
diff --git a/pypy/module/cpyext/c-api.txt b/pypy/module/cpyext/c-api.txt
deleted file mode 100644
--- a/pypy/module/cpyext/c-api.txt
+++ /dev/null
@@ -1,71 +0,0 @@
-Reference Count
-===============
-
-XXX
-
-Borrowed References
-===================
-
-XXX
-
-PyStringObject support
-======================
-
-The problem
------------
-
-PyString_AsString() returns a (non-movable) pointer to the underlying
-buffer, whereas pypy strings are movable.  C code may temporarily
-store this address and use it, as long as it owns a reference to the
-PyObject.  There is no "release" function to specify that the pointer
-is not needed any more.
-
-Note that the pointer may be used to fill the initial value of
-string. This is valid only when the string was just allocated, and is
-not used elsewhere.
-
-Proposed solution
------------------
-
-Our emulation of the PyStringObject contains an additional member: a
-pointer to a char buffer; it may be NULL.
-
-- A string allocated by pypy will be converted into a PyStringObject
-  with a NULL buffer.  When PyString_AsString() is called, memory is
-  allocated (with flavor='raw') and content is copied.
-
-- A string allocated with PyString_FromStringAndSize(NULL, size) will
-  allocate a buffer with the specified size, but the reference won't
-  be stored in the global map py_objects_r2w; there won't be a
-  corresponding object in pypy.  When from_ref() or Py_INCREF() is
-  called, the pypy string is created, and added in py_objects_r2w.
-  The buffer is then supposed to be immutable.
-
-- _PyString_Resize works only on not-yet-pypy'd strings, and returns a
-  similar object.
-
-- PyString_Size don't need to force the object. (in this case, another
-  "size" member is needed)
-
-- There could be an (expensive!) check in from_ref() that the buffer
-  still corresponds to the pypy gc-managed string.
-
-PySequence_Fast support
-======================
-There are five functions for fast sequence access offered by the CPython API:
-
-PyObject* PySequence_Fast(PyObject *o, const char *m)
-
-PyObject* PySequence_Fast_GET_ITEM(	PyObject *o, int i)
-
-PyObject** PySequence_Fast_ITEMS(	PyObject *o)
-
-PyObject* PySequence_ITEM(	PyObject *o, int i)
-
-int PySequence_Fast_GET_SIZE(	PyObject *o)
-
-PyPy supports four of these, but does not support PySequence_Fast_ITEMS.
-(Various ways to support PySequence_Fast_ITEMS were considered. They all had
-two things in common: they would have taken a lot of work, and they would have
-resulted in incomplete semantics or in poor performance. We decided that a slow
-implementation of PySequence_Fast_ITEMS was not very useful.)
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -25,6 +25,8 @@
                          flavor='raw',
                          add_memory_pressure=True)
 
+realloc = rffi.llexternal('realloc', [rffi.VOIDP, rffi.SIZE_T], rffi.VOIDP)
+
 @cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
@@ -32,7 +34,7 @@
                          flavor='raw',
                          add_memory_pressure=True)
     # XXX FIXME
-    return lltype.nullptr(rffi.VOIDP.TO)
+    return realloc(ptr, size)
 
 @cpython_api([rffi.VOIDP], lltype.Void)
 def PyObject_Free(space, ptr):
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -10,7 +10,7 @@
 from pypy.objspace.std import tupleobject
 
 from pypy.module.cpyext.tupleobject import PyTuple_Check, PyTuple_SetItem
-from pypy.module.cpyext.object import Py_IncRef, Py_DecRef
+from pypy.module.cpyext.pyobject import decref
 
 from pypy.module.cpyext.dictobject import PyDict_Check
 
@@ -252,7 +252,7 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        Py_DecRef(w_list.space, storage._elems[index])
+        decref(w_list.space, storage._elems[index])
         storage._elems[index] = make_ref(w_list.space, w_obj)
 
     def length(self, w_list):
@@ -264,9 +264,8 @@
         return storage._elems
 
     def getslice(self, w_list, start, stop, step, length):
-        #storage = self.unerase(w_list.lstorage)
-        raise oefmt(w_list.space.w_NotImplementedError,
-                    "settting a slice of a PySequence_Fast is not supported")
+        w_list.switch_to_object_strategy()
+        return w_list.strategy.getslice(w_list, start, stop, step, length)
 
     def getitems(self, w_list):
         # called when switching list strategy, so convert storage
@@ -389,5 +388,5 @@
 
     def __del__(self):
         for i in range(self._length):
-            Py_DecRef(self.space, self._elems[i])
+            decref(self.space, self._elems[i])
         lltype.free(self._elems, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.eval import (
     Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP
+from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
@@ -130,19 +130,19 @@
     def test_run_file(self, space, api):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
-        fp = fopen(str(filepath), "rb")
+        fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_ZeroDivisionError
         api.PyErr_Clear()
 
         # try again, but with a closed file
-        fp = fopen(str(filepath), "rb")
-        os.close(fileno(fp))
+        fp = c_fopen(str(filepath), "rb")
+        os.close(c_fileno(fp))
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -212,8 +212,9 @@
         assert type(x) is float
         assert x == -12.34
 
-    @pytest.mark.skipif(True, reason='realloc not fully implemented')
     def test_object_realloc(self):
+        if not self.runappdirect:
+            skip('no untranslated support for realloc')
         module = self.import_extension('foo', [
             ("realloctest", "METH_NOARGS",
              """
@@ -221,12 +222,11 @@
                  char *copy, *orig = PyObject_MALLOC(12);
                  memcpy(orig, "hello world", 12);
                  copy = PyObject_REALLOC(orig, 15);
+                 /* realloc() takes care of freeing orig, if changed */
                  if (copy == NULL)
                      Py_RETURN_NONE;
                  ret = PyBytes_FromStringAndSize(copy, 12);
-                 if (copy != orig)
-                     PyObject_Free(copy);
-                 PyObject_Free(orig);
+                 PyObject_Free(copy);
                  return ret;
              """)])
         x = module.realloctest()
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -78,6 +78,17 @@
         assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
         assert space.eq_w(w_t, space.wrap([1, 3, 5]))
 
+    def test_get_slice_fast(self, space, api):
+        w_t = space.wrap([1, 2, 3, 4, 5])
+        api.PySequence_Fast(w_t, "foo") # converts
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4]
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4]
+
+        assert api.PySequence_DelSlice(w_t, 1, 4) == 0
+        assert space.eq_w(w_t, space.wrap([1, 5]))
+        assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
+        assert space.eq_w(w_t, space.wrap([1, 3, 5]))
+
     def test_iter(self, space, api):
         w_t = space.wrap((1, 2))
         w_iter = api.PySeqIter_New(w_t)
@@ -226,18 +237,33 @@
         assert space.int_w(space.len(w_l)) == 10
 
 
-class XAppTestSequenceObject(AppTestCpythonExtensionBase):
-    def test_sequenceobject(self):
+class AppTestSequenceObject(AppTestCpythonExtensionBase):
+    def test_fast(self):
         module = self.import_extension('foo', [
             ("test_fast_sequence", "METH_VARARGS",
              """
-                PyObject * o = PyTuple_GetItem(args, 0);
+                int size, i;
+                PyTypeObject * common_type;
+                PyObject *foo, **objects;
+                PyObject * seq = PyTuple_GetItem(args, 0);
                 /* XXX assert it is a tuple */
-                PyObject *foo = PySequence_Fast(o, "some string");
-                PyObject ** res = PySequence_Fast_ITEMS(foo);
-                /* XXX do some kind of test on res */
-                /* XXX now what? who manages res's refcount? */
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "some string");
+                objects = PySequence_Fast_ITEMS(foo);
+                size = PySequence_Fast_GET_SIZE(seq);
+                common_type = size > 0 ? Py_TYPE(objects[0]) : NULL;
+                for (i = 1; i < size; ++i) {
+                    if (Py_TYPE(objects[i]) != common_type) {
+                        common_type = NULL;
+                        break;
+                    }
+                }
+                Py_DECREF(foo);
+                Py_DECREF(common_type);
                 return PyBool_FromLong(1);
              """)])
-        assert module.test_fast_sequence([1, 2, 3, 4])
+        s = [1, 2, 3, 4]
+        assert module.test_fast_sequence(s[0:-1])
+        assert module.test_fast_sequence(s[::-1])
 
diff --git a/pypy/module/imp/__init__.py b/pypy/module/imp/__init__.py
--- a/pypy/module/imp/__init__.py
+++ b/pypy/module/imp/__init__.py
@@ -2,8 +2,7 @@
 
 class Module(MixedModule):
     """
-    This module provides the components needed to build your own
-    __import__ function.
+    (Extremely) low-level import machinery bits as used by importlib and imp.
     """
     applevel_name = '_imp'
 
@@ -12,7 +11,7 @@
 
         'get_magic':       'interp_imp.get_magic',
         'get_tag':         'interp_imp.get_tag',
-        'load_dynamic':    'interp_imp.load_dynamic',
+        'create_dynamic':  'interp_imp.create_dynamic',
         'create_builtin':  'interp_imp.create_builtin',
         'init_frozen':     'interp_imp.init_frozen',
         'is_builtin':      'interp_imp.is_builtin',
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -117,6 +117,13 @@
         space.sys.setmodule(w_mod)
     return w_mod
 
+def load_c_extension(space, filename, modulename):
+    from pypy.module.cpyext.api import load_extension_module
+    log_pyverbose(space, 1, "import %s # from %s\n" %
+                  (modulename, filename))
+    load_extension_module(space, filename, modulename)
+    # NB. cpyext.api.load_extension_module() can also delegate to _cffi_backend
+
 # __________________________________________________________________
 #
 # import lock, to prevent two threads from running module-level code in
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -50,17 +50,13 @@
         fd = space.int_w(space.call_method(w_iobase, 'fileno'))
         return streamio.fdopen_as_stream(fd, filemode)
 
- at unwrap_spec(filename='fsencode')
-def load_dynamic(space, w_modulename, filename, w_file=None):
+def create_dynamic(space, w_spec, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
-
-    # the next line is mandatory to init cpyext
-    space.getbuiltinmodule("cpyext")
-
-    from pypy.module.cpyext.api import load_extension_module
-    load_extension_module(space, filename, space.str_w(w_modulename))
-
+    w_modulename = space.getattr(w_spec, space.wrap("name"))
+    w_path = space.getattr(w_spec, space.wrap("origin"))
+    filename = space.fsencode_w(w_path)
+    importing.load_c_extension(space, filename, space.str_w(w_modulename))
     return importing.check_sys_modules(space, w_modulename)
 
 def create_builtin(space, w_spec):
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -54,11 +54,13 @@
         finally:
             del sys.path[0]
 
-    def test_load_dynamic(self):
+    def test_create_dynamic(self):
         import imp
-        raises(ImportError, imp.load_dynamic, 'foo', 'bar')
-        raises(ImportError, imp.load_dynamic, 'foo', 'bar',
-               open(self.file_module))
+        class FakeSpec:
+            name = 'foo'
+            origin = 'this/path/does/not/exist'
+        raises(ImportError, imp.create_dynamic, FakeSpec())
+        raises(ImportError, imp.create_dynamic, FakeSpec(), "unused")
 
     def test_suffixes(self):
         import imp
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -894,6 +894,9 @@
         fd1, fd2 = os.pipe()
     except OSError as e:
         raise wrap_oserror(space, e)
+    # XXX later, use rposix.pipe2() if available!
+    rposix.set_inheritable(fd1, False)
+    rposix.set_inheritable(fd2, False)
     return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -47,6 +47,7 @@
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
+    from rpython.tool.error import offset2lineno
     bcindex = ord(bytecode.co_code[next_instr])
     opname = ""
     if 0 <= bcindex < len(opcode_method_names):
@@ -54,7 +55,8 @@
     name = bytecode.co_name
     if not name:
         name = ""
-    return (bytecode.co_filename, bytecode.co_firstlineno,
+    line = offset2lineno(bytecode, intmask(next_instr))
+    return (bytecode.co_filename, line,
             name, intmask(next_instr), opname)
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -19,6 +19,7 @@
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
         self.debug = True
+        self.track_resources = False
         self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -130,7 +130,7 @@
                 cls.module = str(udir.join('testownlib.dll'))
         else:
             subprocess.check_call(
-                'gcc testownlib.c -shared -fPIC -o testownlib.so',
+                'cc testownlib.c -shared -fPIC -o testownlib.so',
                 cwd=str(udir), shell=True)
             cls.module = str(udir.join('testownlib.so'))
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -852,9 +852,12 @@
     assert str(e2.value) == "foo0() takes no arguments (2 given)"
     assert str(e3.value) == "foo1() takes exactly one argument (0 given)"
     assert str(e4.value) == "foo1() takes exactly one argument (2 given)"
-    assert str(e5.value) == "foo2() takes exactly 2 arguments (0 given)"
-    assert str(e6.value) == "foo2() takes exactly 2 arguments (1 given)"
-    assert str(e7.value) == "foo2() takes exactly 2 arguments (3 given)"
+    assert str(e5.value) in ["foo2 expected 2 arguments, got 0",
+                             "foo2() takes exactly 2 arguments (0 given)"]
+    assert str(e6.value) in ["foo2 expected 2 arguments, got 1",
+                             "foo2() takes exactly 2 arguments (1 given)"]
+    assert str(e7.value) in ["foo2 expected 2 arguments, got 3",
+                             "foo2() takes exactly 2 arguments (3 given)"]
 
 def test_address_of_function():
     ffi = FFI()
@@ -1916,3 +1919,47 @@
     ffi.cdef("bool f(void);")
     lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
     assert lib.f() == 1
+
+def test_bool_in_cpp_2():
+    ffi = FFI()
+    ffi.cdef('int add(int a, int b);')
+    lib = verify(ffi, "test_bool_bug_cpp", '''
+        typedef bool _Bool;  /* there is a Windows header with this line */
+        int add(int a, int b)
+        {
+            return a + b;
+        }''', source_extension='.cpp')
+    c = lib.add(2, 3)
+    assert c == 5
+
+def test_struct_field_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[2]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+    ffi = FFI()
+    ffi.cdef("struct a { struct b b[]; };")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_struct_field_opaque", "?")
+    assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+                            " type (not declared in cdef())")
+
+def test_function_arg_opaque():
+    py.test.skip("can currently declare a function with an opaque struct "
+                 "as argument, but AFAICT it's impossible to call it later")
+
+def test_function_returns_opaque():
+    ffi = FFI()
+    ffi.cdef("struct a foo(int);")
+    e = py.test.raises(TypeError, verify,
+                       ffi, "test_function_returns_opaque", "?")
+    assert str(e.value) == ("function foo: 'struct a' is used as result type,"
+                            " but is opaque")
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -133,6 +133,12 @@
         # You cannot assing character format codes as restype any longer
         raises(TypeError, setattr, f, "restype", "i")
 
+    def test_unicode_function_name(self):
+        f = dll[u'_testfunc_i_bhilfd']
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_int
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        assert result == 21
 
     def test_truncate_python_longs(self):
         f = dll._testfunc_i_bhilfd
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -159,7 +159,6 @@
         libraries=rtime.libraries
     )
     CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
@@ -233,7 +232,6 @@
 HAS_CLOCK_MONOTONIC = cConfig.CLOCK_MONOTONIC is not None
 HAS_MONOTONIC = (_WIN or _MACOSX or
                  (HAS_CLOCK_GETTIME and (HAS_CLOCK_HIGHRES or HAS_CLOCK_MONOTONIC)))
-clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
 
@@ -1030,7 +1028,10 @@
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
                 if rffi.cast(lltype.Signed, ret) != -1:
-                    cpu_time = float(tms.c_tms_utime + tms.c_tms_stime)
+                    cpu_time = float(rffi.cast(lltype.Signed,
+                                               tms.c_tms_utime) +
+                                     rffi.cast(lltype.Signed,
+                                               tms.c_tms_stime))
                     if w_info is not None:
                         _setinfo(space, w_info, "times()",
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
@@ -1038,7 +1039,7 @@
                     return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
         return clock(space)
 
-_clock = external('clock', [], clock_t)
+_clock = external('clock', [], rposix.CLOCK_T)
 def clock(space, w_info=None):
     """clock() -> floating point number
 
@@ -1052,7 +1053,7 @@
             pass
     value = _clock()
     # Is this casting correct?
-    if value == rffi.cast(clock_t, -1):
+    if intmask(value) == intmask(rffi.cast(rposix.CLOCK_T, -1)):
         raise oefmt(space.w_RuntimeError,
                     "the processor time used is not available or its value"
                     "cannot be represented")
diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -153,10 +153,12 @@
         S. Out of bounds indices are clipped in a manner consistent with the
         handling of normal slices.
         """
-        length = space.getindex_w(w_length, space.w_OverflowError)
-        start, stop, step = self.indices3(space, length)
-        return space.newtuple([space.wrap(start), space.wrap(stop),
-                               space.wrap(step)])
+        # like CPython 3.5, we duplicate this whole functionality for
+        # this rarely-used method instead of using the existing logic
+        # in indices3(), just to support 'slice(a,b,c).indices(d)' where
+        # all of a, b, c and d are very large integers.
+        return app_indices(space, self.w_start, self.w_stop,
+                           self.w_step, w_length)
 
 
 def slicewprop(name):
@@ -248,3 +250,68 @@
     elif start > stop:
         start = stop
     return start, stop
+
+
+app = gateway.applevel("""
+    from operator import index
+
+    def evaluate_slice_index(x):
+        try:
+            return index(x)
+        except TypeError:
+            raise TypeError("slice indices must be integers or "
+                            "None or have an __index__ method")
+
+    def _getlongindices(start, stop, step, length):
+        if step is None:
+            step = 1
+        else:
+            step = evaluate_slice_index(step)
+            if step == 0:
+                raise ValueError("slice step cannot be zero")
+
+        # Find lower and upper bounds for start and stop.
+        if step < 0:
+            lower = -1
+            upper = length - 1
+        else:
+            lower = 0
+            upper = length
+
+        # Compute start.
+        if start is None:
+            start = upper if step < 0 else lower
+        else:
+            start = evaluate_slice_index(start)
+            if start < 0:
+                start += length
+                if start < lower:
+                    start = lower
+            else:
+                if start > upper:
+                    start = upper
+
+        # Compute stop.
+        if stop is None:
+            stop = lower if step < 0 else upper
+        else:
+            stop = evaluate_slice_index(stop)
+            if stop < 0:
+                stop += length
+                if stop < lower:
+                    stop = lower
+            else:
+                if stop > upper:
+                    stop = upper
+
+        return (start, stop, step)
+
+    def indices(start, stop, step, length):
+        length = index(length)
+        if length < 0:
+            raise ValueError("length should not be negative")
+        return _getlongindices(start, stop, step, length)
+
+""", filename=__file__)
+
+app_indices = app.interphook("indices")
diff --git a/pypy/objspace/std/test/test_sliceobject.py b/pypy/objspace/std/test/test_sliceobject.py
--- a/pypy/objspace/std/test/test_sliceobject.py
+++ b/pypy/objspace/std/test/test_sliceobject.py
@@ -111,11 +111,11 @@
         assert slice(-2 ** 200, -2 ** 100, 1).indices(1000) == (0, 0, 1)
         assert slice(2 ** 100, 0, -1).indices(1000) == (999, 0, -1)
         assert slice(2 ** 100, -2 ** 100, -1).indices(1000) == (999, -1, -1)
-        start, stop, step = slice(0, 1000, 2 ** 200).indices(1000)
-        assert start == 0
-        assert stop == 1000
-        assert step >= 1000
-        raises(OverflowError, "slice(0, 1000, 1).indices(2 ** 100)")
+        assert slice(0, 1000, 2 ** 200).indices(1000) == (0, 1000, 2 ** 200)
+        assert slice(0, 1000, 1).indices(2 ** 100) == (0, 1000, 1)
 
     def test_reduce(self):
         assert slice(1, 2, 3).__reduce__() == (slice, (1, 2, 3))
+
+    def test_indices_negative_length(self):
+        raises(ValueError, "slice(0, 1000, 1).indices(-1)")
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1388,3 +1388,7 @@
         assert not self.compares_by_identity(X)
         del X.__eq__
         assert self.compares_by_identity(X)
+
+    def test_duplicate_slot_name(self):
+        class X:   # does not raise
+            __slots__ = 'a', 'a'
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -930,6 +930,7 @@
         abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj))
 
 def type_get_dict(space, w_cls):
+    w_cls = _check(space, w_cls)
     from pypy.objspace.std.dictproxyobject import W_DictProxyObject
     w_dict = w_cls.getdict(space)
     if w_dict is None:
@@ -1075,9 +1076,14 @@
     # create member
     slot_name = mangle(slot_name, w_self.name)
     if slot_name in w_self.dict_w:
-        raise oefmt(space.w_ValueError,
-                    "%R in __slots__ conflicts with class variable",
-                    w_slot_name)
+        w_prev = w_self.dict_w[slot_name]
+        if isinstance(w_prev, Member) and w_prev.w_cls is w_self:
+            pass   # special case: duplicate __slots__ entry, ignored
+                   # (e.g. occurs in datetime.py, fwiw)
+        else:
+            raise oefmt(space.w_ValueError,
+                        "%R in __slots__ conflicts with class variable",
+                        w_slot_name)
     else:
         # Force interning of slot names.
         slot_name = space.str_w(space.new_interned_str(slot_name))
@@ -1287,7 +1293,8 @@
     cycle.append(candidate)
     cycle.reverse()
     names = [cls.getname(space) for cls in cycle]
-    raise OperationError(space.w_TypeError, space.wrap(
+    # Can't use oefmt() here, since names is a list of unicodes
+    raise OperationError(space.w_TypeError, space.newunicode(
         u"cycle among base classes: " + u' < '.join(names)))
 
 
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -143,3 +143,5 @@
     def is_w(self, obj1, obj2):
         return obj1 is obj2
 
+    def setitem(self, obj, key, value):
+        obj[key] = value
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-# hypothesis is used for test generation on untranslated jit tests
+# hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -401,6 +401,9 @@
 class __extend__(pairtype(SomeString, SomeTuple),
                  pairtype(SomeUnicodeString, SomeTuple)):
     def mod((s_string, s_tuple)):
+        if not s_string.is_constant():
+            raise AnnotatorError("string formatting requires a constant "
+                                 "string/unicode on the left of '%'")
         is_string = isinstance(s_string, SomeString)
         is_unicode = isinstance(s_string, SomeUnicodeString)
         assert is_string or is_unicode
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4623,6 +4623,14 @@
         a = self.RPythonAnnotator()
         a.build_types(main, [int])
 
+    def test_string_mod_nonconstant(self):
+        def f(x):
+            return x % 5
+        a = self.RPythonAnnotator()
+        e = py.test.raises(AnnotatorError, a.build_types, f, [str])
+        assert ('string formatting requires a constant string/unicode'
+                in str(e.value))
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -883,6 +883,7 @@
             ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
                                               self.cpu.translate_support_code)
             assert itemsize == 1
+            ofs_items -= 1     # for the extra null character
             scale = 0
         self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -280,7 +280,7 @@
     concrete_type = '\x00'
 
     def __init__(self, basesize, itemsize, lendescr, flag, is_pure=False, concrete_type='\x00'):
-        self.basesize = basesize
+        self.basesize = basesize    # this includes +1 for STR
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
@@ -676,7 +676,7 @@
 
 def unpack_arraydescr(arraydescr):
     assert isinstance(arraydescr, ArrayDescr)
-    ofs = arraydescr.basesize
+    ofs = arraydescr.basesize    # this includes +1 for STR
     size = arraydescr.itemsize
     sign = arraydescr.is_item_signed()
     return size, ofs, sign
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -293,6 +293,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_load_or_indexed(op, op.getarg(0), op.getarg(1),
                                          itemsize, itemsize, basesize, NOT_SIGNED)
         elif opnum == rop.UNICODEGETITEM:
@@ -304,6 +305,7 @@
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
             assert itemsize == 1
+            basesize -= 1     # for the extra null character
             self.emit_gc_store_or_indexed(op, op.getarg(0), op.getarg(1), op.getarg(2),
                                          itemsize, itemsize, basesize)
         elif opnum == rop.UNICODESETITEM:
diff --git a/rpython/jit/backend/llsupport/symbolic.py b/rpython/jit/backend/llsupport/symbolic.py

From pypy.commits at gmail.com  Wed Aug 17 06:34:56 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 17 Aug 2016 03:34:56 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: add tests,
 implement buffer check
Message-ID: <57b43dd0.a6a5c20a.d3152.7c76@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86242:a0ab08cd0fb3
Date: 2016-08-16 18:47 +0300
http://bitbucket.org/pypy/pypy/changeset/a0ab08cd0fb3/

Log:	add tests, implement buffer check

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -121,7 +121,7 @@
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 """.split()
 for name in constant_names:
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,13 +1,17 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer)
+    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, w_obj):
+def PyObject_CheckBuffer(space, pyobj):
     """Return 1 if obj supports the buffer interface otherwise 0."""
-    return 0  # the bf_getbuffer field is never filled by cpyext
+    as_buffer = pyobj.c_ob_type.c_tp_as_buffer
+    flags = pyobj.c_ob_type.c_tp_flags
+    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.bf_getbuffer):
+        return 1
+    return 0  
 
 @cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
              rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -0,0 +1,237 @@
+#include <Python.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+/* 
+ * Adapted from https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol,
+ * which is copyright Jake Vanderplas and released under the BSD license
+ */
+
+/* Structure defines a 1-dimensional strided array */
+typedef struct{
+    int* arr;
+    long length;
+} MyArray;
+
+/* initialize the array with integers 0...length */
+void initialize_MyArray(MyArray* a, long length){
+    a->length = length;
+    a->arr = (int*)malloc(length * sizeof(int));
+    for(int i=0; i<length; i++){
+        a->arr[i] = i;
+    }
+}
+
+/* free the memory when finished */
+void deallocate_MyArray(MyArray* a){
+    free(a->arr);
+    a->arr = NULL;
+}
+
+/* tools to print the array */
+char* stringify(MyArray* a, int nmax){
+    char* output = (char*) malloc(nmax * 20);
+    int pos = sprintf(&output[0], "[");
+
+    for (int k=0; k < a->length && k < nmax; k++){
+        pos += sprintf(&output[pos], " %d", a->arr[k]);
+    }
+    if(a->length > nmax)
+        pos += sprintf(&output[pos], "...");
+    sprintf(&output[pos], " ]");
+    return output;
+}
+
+void print_MyArray(MyArray* a, int nmax){
+    char* s = stringify(a, nmax);
+    printf("%s", s);
+    free(s);
+}
+
+/* This is where we define the PyMyArray object structure */
+typedef struct {
+    PyObject_HEAD
+    /* Type-specific fields go below. */
+    MyArray arr;
+} PyMyArray;
+
+
+/* This is the __init__ function, implemented in C */
+static int
+PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
+{
+    // init may have already been called
+    if (self->arr.arr != NULL);
+        deallocate_MyArray(&self->arr);
+
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
+    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
+        return -1;
+
+    if (length < 0)
+        length = 0;
+
+    initialize_MyArray(&self->arr, length);
+
+    return 0;
+}
+
+
+/* this function is called when the object is deallocated */
+static void
+PyMyArray_dealloc(PyMyArray* self)
+{
+    deallocate_MyArray(&self->arr);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+
+/* This function returns the string representation of our object */
+static PyObject *
+PyMyArray_str(PyMyArray * self)
+{
+  char* s = stringify(&self->arr, 10);
+  PyObject* ret = PyUnicode_FromString(s);
+  free(s);
+  return ret;
+}
+
+/* Here is the buffer interface function */
+static int
+PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
+{
+  if (view == NULL) {
+    PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
+    return -1;
+  }
+
+  PyMyArray* self = (PyMyArray*)obj;
+  view->obj = (PyObject*)self;
+  view->buf = (void*)self->arr.arr;
+  view->len = self->arr.length * sizeof(int);
+  view->readonly = 0;
+  view->itemsize = sizeof(int);
+  view->format = "i";  // integer
+  view->ndim = 1;
+  view->shape = &self->arr.length;  // length-1 sequence of dimensions
+  view->strides = &view->itemsize;  // for the simple case we can do this
+  view->suboffsets = NULL;
+  view->internal = NULL;
+
+  Py_INCREF(self);  // need to increase the reference count
+  return 0;
+}
+
+static PyBufferProcs PyMyArray_as_buffer = {
+#if PY_MAJOR_VERSION < 3
+  (readbufferproc)0,
+  (writebufferproc)0,
+  (segcountproc)0,
+  (charbufferproc)0,
+#endif
+  (getbufferproc)PyMyArray_getbuffer,
+  (releasebufferproc)0,  // we do not require any special release function
+};
+
+
+/* Here is the type structure: we put the above functions in the appropriate place
+   in order to actually define the Python object type */
+static PyTypeObject PyMyArrayType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "pymyarray.PyMyArray",        /* tp_name */
+    sizeof(PyMyArray),            /* tp_basicsize */
+    0,                            /* tp_itemsize */
+    (destructor)PyMyArray_dealloc,/* tp_dealloc */
+    0,                            /* tp_print */
+    0,                            /* tp_getattr */
+    0,                            /* tp_setattr */
+    0,                            /* tp_reserved */
+    (reprfunc)PyMyArray_str,      /* tp_repr */
+    0,                            /* tp_as_number */
+    0,                            /* tp_as_sequence */
+    0,                            /* tp_as_mapping */
+    0,                            /* tp_hash  */
+    0,                            /* tp_call */
+    (reprfunc)PyMyArray_str,      /* tp_str */
+    0,                            /* tp_getattro */
+    0,                            /* tp_setattro */
+    &PyMyArray_as_buffer,         /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */
+    "PyMyArray object",           /* tp_doc */
+    0,                            /* tp_traverse */
+    0,                            /* tp_clear */
+    0,                            /* tp_richcompare */
+    0,                            /* tp_weaklistoffset */
+    0,                            /* tp_iter */
+    0,                            /* tp_iternext */
+    0,                            /* tp_methods */
+    0,                            /* tp_members */
+    0,                            /* tp_getset */
+    0,                            /* tp_base */
+    0,                            /* tp_dict */
+    0,                            /* tp_descr_get */
+    0,                            /* tp_descr_set */
+    0,                            /* tp_dictoffset */
+    (initproc)PyMyArray_init,     /* tp_init */
+};
+
+static PyMethodDef buffer_functions[] = {
+    {NULL,        NULL}    /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "buffer_test",
+    "Module Doc",
+    -1,
+    buffer_functions;
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+};
+#define INITERROR return NULL
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+PyInit_buffer_test(void)
+
+#else
+
+#define INITERROR return
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+initbuffer_test(void)
+#endif
+{
+#if PY_MAJOR_VERSION >= 3
+    PyObject *m= PyModule_Create(&moduledef);
+#else
+    PyObject *m= Py_InitModule("buffer_test", buffer_functions);
+#endif
+    if (m == NULL)
+        INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&PyMyArrayType) < 0)
+        INITERROR;
+    Py_INCREF(&PyMyArrayType);
+    PyModule_AddObject(m, "PyMyArray", (PyObject *)&PyMyArrayType);
+#if PY_MAJOR_VERSION >=3
+    return m;
+#endif
+}
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,5 +1,8 @@
 import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
@@ -8,10 +11,16 @@
             py.test.skip("unsupported before Python 2.7")
 
         w_hello = space.newbytes("hello")
+        assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
-    @pytest.mark.skipif(True, reason='write a test for this')
-    def test_get_base_and_get_buffer(self, space, api):
-        assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER
+
+class AppTestBufferProtocol(AppTestCpythonExtensionBase):
+    def test_buffer_protocol(self):
+        module = self.import_module(name='buffer_test')
+        arr = module.PyMyArray(10)
+        y = memoryview(arr)
+        assert y.format == 'i'
+

From pypy.commits at gmail.com  Wed Aug 17 06:58:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 03:58:34 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: update
Message-ID: <57b4435a.482cc20a.85a96.7acf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5672:e13c7d0db892
Date: 2016-08-17 12:58 +0200
http://bitbucket.org/pypy/extradoc/changeset/e13c7d0db892/

Log:	update

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -6,9 +6,8 @@
 
 * Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951 (plan_rich)
 
-* Make a translated py3.5 actually work a bit (currently we get
-  systematic failures), up to the point where we can meaningfully
-  run the lib-python tests (arigo)
+* At some point, review lib-python/conftest.py to remove the skips
+  due to deadlocks (search for "XXX:")
 
 Finished
 --------

From pypy.commits at gmail.com  Wed Aug 17 07:04:39 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 17 Aug 2016 04:04:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Add StopAsyncIteration exception,
 expand test to check 'async for' in test_asyncio
Message-ID: <57b444c7.17a61c0a.31f27.c6b4@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86243:7d96e4bc57bb
Date: 2016-08-17 13:04 +0200
http://bitbucket.org/pypy/pypy/changeset/7d96e4bc57bb/

Log:	Add StopAsyncIteration exception, expand test to check 'async for'
	in test_asyncio

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -619,7 +619,7 @@
         self.emit_jump(ops.JUMP_FORWARD, b_after_try)
         self.use_next_block(b_except)
         self.emit_op(ops.POP_TOP)
-        self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopIterError")
+        self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopAsyncIteration")
         self.emit_op_arg(ops.COMPARE_OP, 10)
         self.emit_jump(ops.POP_JUMP_IF_FALSE, b_try_cleanup, True)
         
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -1,4 +1,5 @@
 class AppTestAsyncIO(object):
+    """These tests are based on the async-await syntax of Python 3.5."""
     
     spaceconfig = dict(usemodules=["select","_socket","thread","signal",
                                    "struct","_multiprocessing","array",
@@ -9,14 +10,19 @@
         # the problem occured at await asyncio.open_connection
         # after calling run_until_complete
         """
-        import encodings.idna
-        import asyncio
-        async def f():
-            reader, writer = await asyncio.open_connection('example.com', 80)
-        
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(f())
-        print("done with async loop")
+import encodings.idna
+import asyncio
+
+async def f():
+    reader, writer = await asyncio.open_connection('example.com', 80)
+    writer.write(b'a')
+    async for line in reader:
+        print('>>>', line)
+    writer.close()
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(f())
+print("done with async loop")
         """
     
     def test_asynchronous_context_managers(self):
diff --git a/pypy/module/exceptions/__init__.py b/pypy/module/exceptions/__init__.py
--- a/pypy/module/exceptions/__init__.py
+++ b/pypy/module/exceptions/__init__.py
@@ -51,6 +51,7 @@
         'ResourceWarning'  : 'interp_exceptions.W_ResourceWarning',
         'RuntimeError' : 'interp_exceptions.W_RuntimeError',
         'RuntimeWarning' : 'interp_exceptions.W_RuntimeWarning',
+        'StopAsyncIteration' : 'interp_exceptions.W_StopAsyncIteration',
         'StopIteration' : 'interp_exceptions.W_StopIteration',
         'SyntaxError' : 'interp_exceptions.W_SyntaxError',
         'SyntaxWarning' : 'interp_exceptions.W_SyntaxWarning',
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -64,6 +64,7 @@
       +-- ReferenceError
       +-- RuntimeError
       |    +-- NotImplementedError
+      +-- StopAsyncIteration
       +-- SyntaxError
       |    +-- IndentationError
       |         +-- TabError
@@ -833,6 +834,9 @@
 W_AssertionError = _new_exception('AssertionError', W_Exception,
                                   """Assertion failed.""")
 
+W_StopAsyncIteration = _new_exception('StopAsyncIteration', W_Exception,
+                                  """Signal the end from iterator.__anext__().""")
+
 class W_UnicodeDecodeError(W_UnicodeError):
     """Unicode decoding error."""
     w_encoding = None

From pypy.commits at gmail.com  Wed Aug 17 07:18:21 2016
From: pypy.commits at gmail.com (Raemi)
Date: Wed, 17 Aug 2016 04:18:21 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: a branch to experiment with a new
 GC (see TODO file)
Message-ID: <57b447fd.469d1c0a.69927.cc60@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: quad-color-gc
Changeset: r86244:8ca6fccd6b2f
Date: 2016-08-17 13:13 +0200
http://bitbucket.org/pypy/pypy/changeset/8ca6fccd6b2f/

Log:	a branch to experiment with a new GC (see TODO file)

diff --git a/TODO b/TODO
new file mode 100644
--- /dev/null
+++ b/TODO
@@ -0,0 +1,10 @@
+Branch for experimenting with N. Truessel's implementation of a generational,
+incremental, non-moving quad-color GC.
+
+Original idea: http://wiki.luajit.org/New-Garbage-Collector
+
+Repository for the actual GC:
+https://github.com/ntruessel/qcgc
+
+
+* TODO document branch

From pypy.commits at gmail.com  Wed Aug 17 07:22:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 04:22:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix the test list,
 and skip for now some that seem to deadlock
Message-ID: <57b448e9.a710c20a.cc582.9dfa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86245:fe7c2ea3520e
Date: 2016-08-17 13:21 +0200
http://bitbucket.org/pypy/pypy/changeset/fe7c2ea3520e/

Log:	Fix the test list, and skip for now some that seem to deadlock

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -99,14 +99,16 @@
     RegrTest('test___all__.py', core=True),
     RegrTest('test___future__.py', core=True),
     RegrTest('test__locale.py', usemodules='_locale'),
+    RegrTest('test__opcode.py'),
     RegrTest('test__osx_support.py'),
     RegrTest('test_abc.py'),
     RegrTest('test_abstract_numbers.py'),
     RegrTest('test_aifc.py'),
     RegrTest('test_argparse.py', usemodules='binascii'),
     RegrTest('test_array.py', core=True, usemodules='struct array binascii'),
+    RegrTest('test_asdl_parser.py'),
     RegrTest('test_ast.py', core=True, usemodules='struct'),
-    RegrTest('test_asynchat.py', usemodules='select fcntl'),
+    RegrTest('test_asynchat.py', usemodules='select fcntl', skip="XXX: deadlocks or takes forever"),
     RegrTest('test_asyncore.py', usemodules='select fcntl'),
     RegrTest('test_atexit.py', core=True),
     RegrTest('test_audioop.py'),
@@ -123,7 +125,7 @@
     RegrTest('test_bufio.py', core=True),
     RegrTest('test_builtin.py', core=True, usemodules='binascii'),
     RegrTest('test_bytes.py', usemodules='struct binascii'),
-    RegrTest('test_bz2.py', usemodules='bz2'),
+    RegrTest('test_bz2.py', usemodules='bz2', skip="XXX: deadlocks?"),
     RegrTest('test_calendar.py'),
     RegrTest('test_call.py', core=True),
     RegrTest('test_capi.py', usemodules='cpyext'),
@@ -134,7 +136,7 @@
     RegrTest('test_cmath.py', core=True),
     RegrTest('test_cmd.py'),
     RegrTest('test_cmd_line.py'),
-    RegrTest('test_cmd_line_script.py'),
+    RegrTest('test_cmd_line_script.py', skip="XXX: deadlocks?"),
     RegrTest('test_code.py', core=True),
     RegrTest('test_code_module.py'),
     RegrTest('test_codeccallbacks.py', core=True),
@@ -151,20 +153,19 @@
     RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'),
     RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec struct unicodedata array'),
     RegrTest('test_codeop.py', core=True),
-    RegrTest('test_coding.py', core=True),
     RegrTest('test_collections.py', usemodules='binascii struct'),
     RegrTest('test_colorsys.py'),
     RegrTest('test_compare.py', core=True),
     RegrTest('test_compile.py', core=True),
     RegrTest('test_compileall.py'),
     RegrTest('test_complex.py', core=True),
-    RegrTest('test_concurrent_futures.py',
-             skip="XXX: deadlocks" if sys.platform == 'win32' else False),
+    RegrTest('test_concurrent_futures.py', skip="XXX: deadlocks" if sys.platform == 'win32' else False),
     RegrTest('test_configparser.py'),
     RegrTest('test_contains.py', core=True),
     RegrTest('test_contextlib.py', usemodules="thread"),
     RegrTest('test_copy.py', core=True),
     RegrTest('test_copyreg.py', core=True),
+    RegrTest('test_coroutines.py'),
     RegrTest('test_cprofile.py'),
     RegrTest('test_crashers.py'),
     RegrTest('test_crypt.py', usemodules='crypt'),
@@ -176,7 +177,7 @@
     RegrTest('test_dbm_dumb.py'),
     RegrTest('test_dbm_gnu.py'),
     RegrTest('test_dbm_ndbm.py'),
-    RegrTest('test_decimal.py'),
+    RegrTest('test_decimal.py', skip="XXX: deadlocks?"),
     RegrTest('test_decorators.py', core=True),
     RegrTest('test_defaultdict.py', usemodules='_collections'),
     RegrTest('test_deque.py', core=True, usemodules='_collections struct'),
@@ -195,8 +196,11 @@
     RegrTest('test_dummy_thread.py', core=True),
     RegrTest('test_dummy_threading.py', core=True),
     RegrTest('test_dynamic.py'),
+    RegrTest('test_dynamicclassattribute.py'),
+    RegrTest('test_eintr.py'),
     RegrTest('test_email', skip="XXX is a directory"),
     RegrTest('test_ensurepip.py'),
+    RegrTest('test_enum.py'),
     RegrTest('test_enumerate.py', core=True),
     RegrTest('test_eof.py', core=True),
     RegrTest('test_epoll.py'),
@@ -204,23 +208,24 @@
     RegrTest('test_exception_variations.py'),
     RegrTest('test_exceptions.py', core=True),
     RegrTest('test_extcall.py', core=True),
-    RegrTest('test_faulthandler.py'),
+    RegrTest('test_faulthandler.py', skip="XXX: deadlocks?"),
     RegrTest('test_fcntl.py', usemodules='fcntl'),
     RegrTest('test_file.py', usemodules="posix", core=True),
     RegrTest('test_file_eintr.py'),
     RegrTest('test_filecmp.py', core=True),
     RegrTest('test_fileinput.py', core=True),
     RegrTest('test_fileio.py'),
+    RegrTest('test_finalization.py'),
     RegrTest('test_float.py', core=True),
     RegrTest('test_flufl.py'),
     RegrTest('test_fnmatch.py', core=True),
-    RegrTest('test_fork1.py', usemodules="thread"),
+    RegrTest('test_fork1.py', usemodules="thread", skip="XXX: deadlocks?"),
     RegrTest('test_format.py', core=True),
     RegrTest('test_fractions.py'),
-    RegrTest('test_frozen.py', skip="unsupported extension module"),
-    RegrTest('test_ftplib.py'),
+    RegrTest('test_frame.py'),
+    RegrTest('test_ftplib.py', skip="XXX: deadlocks?"),
     RegrTest('test_funcattrs.py', core=True),
-    RegrTest('test_functools.py'),
+    RegrTest('test_functools.py', skip="XXX: deadlocks?"),
     RegrTest('test_future.py', core=True),
     RegrTest('test_future3.py', core=True),
     RegrTest('test_future4.py', core=True),
@@ -250,11 +255,9 @@
     RegrTest('test_httplib.py'),
     RegrTest('test_httpservers.py'),
     RegrTest('test_idle.py'),
-    RegrTest('test_imaplib.py'),
+    RegrTest('test_imaplib.py', skip="XXX: deadlocks?"),
     RegrTest('test_imghdr.py'),
     RegrTest('test_imp.py', core=True, usemodules='thread'),
-    RegrTest('test_import.py', core=True),
-    RegrTest('test_importhooks.py', core=True),
     RegrTest('test_importlib', 'XXX is a directory'),
     RegrTest('test_index.py'),
     RegrTest('test_inspect.py', usemodules="struct unicodedata"),
@@ -268,6 +271,7 @@
     RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"),
     RegrTest('test_itertools.py', core=True, usemodules="itertools struct"),
     RegrTest('test_json', skip="XXX is a directory"),
+    RegrTest('test_keyword.py'),
     RegrTest('test_keywordonlyarg.py'),
     RegrTest('test_kqueue.py'),
     RegrTest('test_largefile.py'),
@@ -296,8 +300,10 @@
     RegrTest('test_modulefinder.py'),
     RegrTest('test_msilib.py'),
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
-    RegrTest('test_multiprocessing.py', skip="XXX: deadlocks the buildbots"),
-    RegrTest('test_namespace_pkgs.py'),
+    RegrTest('test_multiprocessing_fork.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_forkserver.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_main_handling.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_spawn.py', skip="XXX: deadlocks?"),
     RegrTest('test_netrc.py'),
     RegrTest('test_nis.py'),
     RegrTest('test_nntplib.py'),
@@ -312,10 +318,10 @@
     RegrTest('test_ossaudiodev.py'),
     RegrTest('test_osx_env.py'),
     RegrTest('test_parser.py', skip="slowly deprecating compiler"),
-    RegrTest('test_pdb.py'),
+    RegrTest('test_pathlib.py'),
+    RegrTest('test_pdb.py', skip="XXX: deadlocks?"),
     RegrTest('test_peepholer.py'),
     RegrTest('test_pep247.py'),
-    RegrTest('test_pep263.py'),
     RegrTest('test_pep277.py'),
     RegrTest('test_pep292.py'),
     RegrTest('test_pep3120.py'),
@@ -323,6 +329,7 @@
     RegrTest('test_pep3151.py'),
     RegrTest('test_pep352.py'),
     RegrTest('test_pep380.py'),
+    RegrTest('test_pep479.py'),
     RegrTest('test_pickle.py', core=True),
     RegrTest('test_pickletools.py', core=False),
     RegrTest('test_pipes.py'),
@@ -331,9 +338,9 @@
     RegrTest('test_pkgutil.py'),
     RegrTest('test_platform.py'),
     RegrTest('test_plistlib.py'),
-    RegrTest('test_poll.py'),
+    RegrTest('test_poll.py', skip="XXX: deadlocks?"),
     RegrTest('test_popen.py'),
-    RegrTest('test_poplib.py'),
+    RegrTest('test_poplib.py', skip="XXX: deadlocks?"),
     RegrTest('test_posix.py', usemodules="_rawffi"),
     RegrTest('test_posixpath.py'),
     RegrTest('test_pow.py', core=True),
@@ -356,6 +363,7 @@
     RegrTest('test_range.py', core=True),
     RegrTest('test_re.py', core=True),
     RegrTest('test_readline.py'),
+    RegrTest('test_regrtest.py'),
     RegrTest('test_reprlib.py', core=True),
     RegrTest('test_resource.py'),
     RegrTest('test_richcmp.py', core=True),
@@ -365,7 +373,9 @@
     RegrTest('test_sax.py'),
     RegrTest('test_sched.py'),
     RegrTest('test_scope.py', core=True),
+    RegrTest('test_script_helper.py'),
     RegrTest('test_select.py'),
+    RegrTest('test_selectors.py'),
     RegrTest('test_set.py', core=True),
     RegrTest('test_setcomps.py', core=True),
     RegrTest('test_shelve.py'),
@@ -381,10 +391,13 @@
     RegrTest('test_socket.py', usemodules='thread _weakref'),
     RegrTest('test_socketserver.py', usemodules='thread'),
     RegrTest('test_sort.py', core=True),
-    RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"),
+    RegrTest('test_source_encoding.py'),
+    RegrTest('test_spwd.py'),
+    RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib", skip="XXX: deadlocks?"),
     RegrTest('test_ssl.py', usemodules='_ssl _socket select'),
     RegrTest('test_startfile.py'),
     RegrTest('test_stat.py'),
+    RegrTest('test_statistics.py'),
     RegrTest('test_strftime.py'),
     RegrTest('test_string.py', core=True),
     RegrTest('test_stringprep.py'),
@@ -394,13 +407,13 @@
     RegrTest('test_struct.py', usemodules='struct'),
     RegrTest('test_structmembers.py', skip="CPython specific"),
     RegrTest('test_structseq.py'),
-    RegrTest('test_subprocess.py', usemodules='signal'),
+    RegrTest('test_subprocess.py', usemodules='signal', skip="XXX: deadlocks?"),
     RegrTest('test_sunau.py'),
     RegrTest('test_sundry.py'),
     RegrTest('test_super.py', core=True),
     RegrTest('test_support.py'),
     RegrTest('test_symtable.py', skip="implementation detail"),
-    RegrTest('test_syntax.py', core=True),
+    RegrTest('test_syntax.py', core=True, skip="XXX: infinite loop?"),
     RegrTest('test_sys.py', core=True, usemodules='struct'),
     RegrTest('test_sys_setprofile.py', core=True),
     RegrTest('test_sys_settrace.py', core=True),
@@ -408,29 +421,30 @@
     RegrTest('test_syslog.py'),
     RegrTest('test_tarfile.py'),
     RegrTest('test_tcl.py'),
-    RegrTest('test_telnetlib.py'),
+    RegrTest('test_telnetlib.py', skip="XXX: deadlocks?"),
     RegrTest('test_tempfile.py'),
     RegrTest('test_textwrap.py'),
     RegrTest('test_thread.py', usemodules="thread", core=True),
-    RegrTest('test_threaded_import.py', usemodules="thread", core=True),
-    RegrTest('test_threadedtempfile.py',
-             usemodules="thread", core=False),
-    RegrTest('test_threading.py', usemodules="thread", core=True),
-    RegrTest('test_threading_local.py', usemodules="thread", core=True),
+    RegrTest('test_threaded_import.py', usemodules="thread", core=True, skip="XXX: deadlocks?"),
+    RegrTest('test_threadedtempfile.py', usemodules="thread", core=False, skip="XXX: deadlocks?"),
+    RegrTest('test_threading.py', usemodules="thread", core=True, skip="XXX: deadlocks?"),
+    RegrTest('test_threading_local.py', usemodules="thread", core=True, skip="XXX: deadlocks?"),
     RegrTest('test_threadsignals.py', usemodules="thread"),
     RegrTest('test_time.py', core=True, usemodules="struct thread _rawffi"),
     RegrTest('test_timeit.py'),
     RegrTest('test_timeout.py'),
+    RegrTest('test_tix.py'),
     RegrTest('test_tk.py'),
     RegrTest('test_tokenize.py'),
-    RegrTest('test_tools.py'),
     RegrTest('test_trace.py'),
     RegrTest('test_traceback.py', core=True),
+    RegrTest('test_tracemalloc.py'),
     RegrTest('test_ttk_guionly.py'),
     RegrTest('test_ttk_textonly.py'),
     RegrTest('test_tuple.py', core=True),
     RegrTest('test_typechecks.py'),
     RegrTest('test_types.py', core=True),
+    RegrTest('test_typing.py'),
     RegrTest('test_ucn.py'),
     RegrTest('test_unary.py', core=True),
     RegrTest('test_unicode.py', core=True),
@@ -455,7 +469,6 @@
     RegrTest('test_venv.py', usemodules="struct"),
     RegrTest('test_wait3.py', usemodules="thread"),
     RegrTest('test_wait4.py', usemodules="thread"),
-    RegrTest('test_warnings.py', core=True),
     RegrTest('test_wave.py'),
     RegrTest('test_weakref.py', core=True, usemodules='_weakref'),
     RegrTest('test_weakset.py'),
@@ -470,8 +483,9 @@
     RegrTest('test_xml_etree_c.py'),
     RegrTest('test_xmlrpc.py'),
     RegrTest('test_xmlrpc_net.py'),
+    RegrTest('test_zipapp.py'),
     RegrTest('test_zipfile.py'),
-    RegrTest('test_zipfile64.py'),
+    RegrTest('test_zipfile64.py', skip="XXX: infinite loop or just long time?"),
     RegrTest('test_zipimport.py', usemodules='zlib zipimport'),
     RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'),
     RegrTest('test_zlib.py', usemodules='zlib'),

From pypy.commits at gmail.com  Wed Aug 17 08:22:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 05:22:39 -0700 (PDT)
Subject: [pypy-commit] cffi default: Document some common mistakes of bad
 lifetime of ffi.new()
Message-ID: <57b4570f.cb7f1c0a.dff6f.e64d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2738:f634d78ac712
Date: 2016-08-17 14:22 +0200
http://bitbucket.org/cffi/cffi/changeset/f634d78ac712/

Log:	Document some common mistakes of bad lifetime of ffi.new()

diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -81,6 +81,33 @@
         # away, then the weak dictionary entry will be removed.
         return s1
 
+Usually you don't need a weak dict: for example, to call a function with
+a ``char * *`` argument that contains a pointer to a ``char *`` pointer,
+it is enough to do this:
+
+.. code-block:: python
+
+    p = ffi.new("char[]", "hello, world")    # p is a 'char *'
+    q = ffi.new("char **", p)                # q is a 'char **'
+    lib.myfunction(q)
+    # p is alive at least until here, so that's fine
+
+However, this is always wrong (usage of freed memory):
+
+.. code-block:: python
+
+    p = ffi.new("char **", ffi.new("char[]", "hello, world"))
+    # WRONG!  as soon as p is built, the inner ffi.new() gets freed!
+
+This is wrong too, for the same reason:
+
+.. code-block:: python
+
+    p = ffi.new("struct my_stuff")
+    p.foo = ffi.new("char[]", "hello, world")
+    # WRONG!  as soon as p.foo is set, the ffi.new() gets freed!
+
+
 The cdata objects support mostly the same operations as in C: you can
 read or write from pointers, arrays and structures.  Dereferencing a
 pointer is done usually in C with the syntax ``*p``, which is not valid

From pypy.commits at gmail.com  Wed Aug 17 08:48:44 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 17 Aug 2016 05:48:44 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: update planned features
Message-ID: <57b45d2c.c186c20a.6dddb.b4f9@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5673:639489d29314
Date: 2016-08-17 10:48 +0200
http://bitbucket.org/pypy/extradoc/changeset/639489d29314/

Log:	update planned features

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -5,6 +5,11 @@
 -------
 
 * Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951 (plan_rich)
+  Seems to work, but test suite hangs to verify the CPython tests.
+* tuple indexing for memory view (plan_rich)
+  Comments: Stronly tied to numpy. Hard to implement, because most of the basics are missing (dimensions/strides)
+  We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
+  Matti's opinion on that would be great!
 
 * At some point, review lib-python/conftest.py to remove the skips
   due to deadlocks (search for "XXX:")
@@ -13,7 +18,6 @@
 --------
 
 
-
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------
 

From pypy.commits at gmail.com  Wed Aug 17 11:22:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 08:22:05 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: A branch to simplify
 asmmemmgr.py so that it is only used for code, not
Message-ID: <57b4811d.a111c20a.22d77.f276@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: asmmemmgr-for-code-only
Changeset: r86248:0194da5a333a
Date: 2016-08-17 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/0194da5a333a/

Log:	A branch to simplify asmmemmgr.py so that it is only used for code,
	not for data


From pypy.commits at gmail.com  Wed Aug 17 11:22:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 08:22:07 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: in-progress
Message-ID: <57b4811f.17a61c0a.31f27.2b03@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: asmmemmgr-for-code-only
Changeset: r86249:9fb620098ee2
Date: 2016-08-17 17:21 +0200
http://bitbucket.org/pypy/pypy/changeset/9fb620098ee2/

Log:	in-progress

diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -29,14 +29,17 @@
         """Returns stats for rlib.jit.jit_hooks.stats_asmmemmgr_*()."""
         return (self.total_memory_allocated, self.total_mallocs)
 
-    def malloc(self, minsize, maxsize):
+    def malloc_code(self, size):
         """Allocate executable memory, between minsize and maxsize bytes,
         and return a pair (start, stop).  Does not perform any rounding
-        of minsize and maxsize.
+        of 'size'; the interesting property is that if all calls to
+        malloc_code() are done with a size that is a multiple of 2**N,
+        then they also return (start, stop) pointers that are aligned
+        to 2**N.
         """
-        result = self._allocate_block(minsize)
+        result = self._allocate_block(size)
         (start, stop) = result
-        smaller_stop = start + maxsize
+        smaller_stop = start + size
         if smaller_stop + self.min_fragment <= stop:
             self._add_free_block(smaller_stop, stop)
             stop = smaller_stop
@@ -44,28 +47,12 @@
         self.total_mallocs += r_uint(stop - start)
         return result   # pair (start, stop)
 
-    def free(self, start, stop):
+    def free_code(self, start, stop):
         """Free a block (start, stop) returned by a previous malloc()."""
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
 
-    def open_malloc(self, minsize):
-        """Allocate at least minsize bytes.  Returns (start, stop)."""
-        result = self._allocate_block(minsize)
-        (start, stop) = result
-        self.total_mallocs += r_uint(stop - start)
-        return result
-
-    def open_free(self, middle, stop):
-        """Used for freeing the end of an open-allocated block of memory."""
-        if stop - middle >= self.min_fragment:
-            self.total_mallocs -= r_uint(stop - middle)
-            self._add_free_block(middle, stop)
-            return True
-        else:
-            return False    # too small to record
-
     def _allocate_large_block(self, minsize):
         # Compute 'size' from 'minsize': it must be rounded up to
         # 'large_alloc_size'.  Additionally, we use the following line
@@ -163,40 +150,6 @@
         self._allocated = None
 
 
-class MachineDataBlockWrapper(object):
-    def __init__(self, asmmemmgr, allblocks):
-        self.asmmemmgr = asmmemmgr
-        self.allblocks = allblocks
-        self.rawstart    = 0
-        self.rawposition = 0
-        self.rawstop     = 0
-
-    def done(self):
-        if self.rawstart != 0:
-            if self.asmmemmgr.open_free(self.rawposition, self.rawstop):
-                self.rawstop = self.rawposition
-            self.allblocks.append((self.rawstart, self.rawstop))
-            self.rawstart    = 0
-            self.rawposition = 0
-            self.rawstop     = 0
-
-    def _allocate_next_block(self, minsize):
-        self.done()
-        self.rawstart, self.rawstop = self.asmmemmgr.open_malloc(minsize)
-        self.rawposition = self.rawstart
-
-    def malloc_aligned(self, size, alignment):
-        p = self.rawposition
-        p = (p + alignment - 1) & (-alignment)
-        if p + size > self.rawstop:
-            self._allocate_next_block(size + alignment - 1)
-            p = self.rawposition
-            p = (p + alignment - 1) & (-alignment)
-            assert p + size <= self.rawstop
-        self.rawposition = p + size
-        return p
-
-
 class BlockBuilderMixin(object):
     _mixin_ = True
     # A base class to generate assembler.  It is equivalent to just a list
@@ -321,11 +274,16 @@
     def materialize(self, cpu, allblocks, gcrootmap=None):
         size = self.get_relative_pos()
         align = self.ALIGN_MATERIALIZE
-        size += align - 1
-        malloced = cpu.asmmemmgr.malloc(size, size)
-        allblocks.append(malloced)
+        size = (size + align - 1) & ~(align - 1)   # round up
+        malloced = cpu.asmmemmgr.malloc_code(size)
         rawstart = malloced[0]
-        rawstart = (rawstart + align - 1) & (-align)
+        rawstop = malloced[1]
+        assert (rawstart & (align - 1)) == 0, (
+            "malloc_code() not aligned to a multiple of ALIGN_MATERIALIZE")
+        assert (rawstart & 1) == 0
+        assert (rawstop & 1) == 0
+        allblocks.append(rawstart)
+        allblocks.append(rawstop - 1)
         self.rawstart = rawstart
         self.copy_to_raw_memory(rawstart)
         if self.gcroot_markers is not None:
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -177,6 +177,21 @@
             clt.asmmemmgr_gcreftracers = []
         return clt.asmmemmgr_gcreftracers
 
+    def malloc_aligned(self, size, alignment=WORD):
+        p1 = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw',
+                           track_allocation=False)
+        s1 = s2 = rffi.cast(lltype.Signed, p1)
+        if (s1 & (alignment - 1)) != 0:   # bah, try again
+            lltype.free(p1, flavor='raw')
+            p1 = lltype.malloc(rffi.CCHARP.TO, size + (alignment - 1),
+                               flavor='raw')
+            s1 = s2 = rffi.cast(lltype.Signed, p1)
+            s2 = (s2 + alignment - 1) & ~(alignment - 1)
+        assert self.allblocks is not None
+        assert (s1 & 1) == 0   # must be even
+        self.allblocks.append(s1)
+        return s2
+
     def set_debug(self, v):
         r = self._debug
         self._debug = v
@@ -491,7 +506,6 @@
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         return bool(gcrootmap) and not gcrootmap.is_shadow_stack
 
-
 def debug_bridge(descr_number, rawstart, codeendpos):
     debug_start("jit-backend-addr")
     debug_print("bridge out of Guard 0x%x has address 0x%x to 0x%x" %
diff --git a/rpython/jit/backend/llsupport/gcmap.py b/rpython/jit/backend/llsupport/gcmap.py
--- a/rpython/jit/backend/llsupport/gcmap.py
+++ b/rpython/jit/backend/llsupport/gcmap.py
@@ -7,8 +7,7 @@
 def allocate_gcmap(assembler, frame_depth, fixed_size):
     size = frame_depth + fixed_size
     malloc_size = (size // WORD // 8 + 1) + 1
-    rawgcmap = assembler.datablockwrapper.malloc_aligned(WORD * malloc_size,
-                                                    WORD)
+    rawgcmap = assembler.malloc_aligned(WORD * malloc_size)
     # set the length field
     rffi.cast(rffi.CArrayPtr(lltype.Signed), rawgcmap)[0] = malloc_size - 1
     gcmap = rffi.cast(lltype.Ptr(jitframe.GCMAP), rawgcmap)
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -256,11 +256,23 @@
         blocks = compiled_loop_token.asmmemmgr_blocks
         if blocks is not None:
             compiled_loop_token.asmmemmgr_blocks = None
-            for rawstart, rawstop in blocks:
-                self.gc_ll_descr.freeing_block(rawstart, rawstop)
-                self.asmmemmgr.free(rawstart, rawstop)
-                if self.HAS_CODEMAP:
-                    self.codemap.free_asm_block(rawstart, rawstop)
+            # see the description in ../model.py about asmmemmgr_blocks
+            i = len(blocks)
+            while i > 0:
+                i -= 1
+                pp = blocks[i]
+                if pp & 1:   # odd number, that's the stop after a start
+                    rawstop = pp + 1
+                    i -= 1
+                    assert i >= 0
+                    rawstart = blocks[i]
+                    self.gc_ll_descr.freeing_block(rawstart, rawstop)
+                    self.asmmemmgr.free_code(rawstart, rawstop)
+                    if self.HAS_CODEMAP:
+                        self.codemap.free_asm_block(rawstart, rawstop)
+                else:
+                    lltype.free(rffi.cast(rffi.CCHARP, pp), flavor='raw',
+                                track_allocation=False)
 
     def force(self, addr_of_force_token):
         frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token)
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -1,6 +1,5 @@
 import random
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
 from rpython.jit.backend.llsupport.codemap import CodemapStorage
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -63,16 +62,15 @@
                               num_indices=5)
     memmgr._add_free_block(10, 18)
     memmgr._add_free_block(20, 30)
-    for minsize in range(1, 11):
-        for maxsize in range(minsize, 14):
-            (start, stop) = memmgr.malloc(minsize, maxsize)
-            if minsize <= 8:
-                assert (start, stop) == (10, 18)
-            else:
-                assert (start, stop) == (20, 30)
-            memmgr._add_free_block(start, stop)
+    for size in range(1, 11):
+        (start, stop) = memmgr.malloc_code(size)
+        if size <= 8:
+            assert (start, stop) == (10, 18)
+        else:
+            assert (start, stop) == (20, 30)
+        memmgr._add_free_block(start, stop)
     memmgr._add_free_block(40, 49)
-    (start, stop) = memmgr.malloc(10, 10)
+    (start, stop) = memmgr.malloc_code(10)
     assert (start, stop) == (20, 30)
 
 def test_malloc_with_fragment():
@@ -80,7 +78,7 @@
         memmgr = AsmMemoryManager(min_fragment=8,
                                   num_indices=5)
         memmgr._add_free_block(12, 44)
-        (start, stop) = memmgr.malloc(reqsize, reqsize)
+        (start, stop) = memmgr.malloc_code(reqsize)
         if reqsize + 8 <= 32:
             assert (start, stop) == (12, 12 + reqsize)
             assert memmgr.free_blocks == {stop: 44}
@@ -108,7 +106,7 @@
         for i in range(100):
             while self.asmmemmgr.total_memory_allocated < 16384:
                 reqsize = random.randrange(1, 200)
-                (start, stop) = self.asmmemmgr.malloc(reqsize, reqsize)
+                (start, stop) = self.asmmemmgr.malloc_code(reqsize)
                 assert reqsize <= stop - start < reqsize + 8
                 assert self.asmmemmgr.total_memory_allocated in [8192, 16384]
             self.teardown_method(None)
@@ -124,19 +122,15 @@
             if got and (random.random() < 0.4 or len(got) == 1000):
                 # free
                 start, stop = got.pop(random.randrange(0, len(got)))
-                self.asmmemmgr.free(start, stop)
+                self.asmmemmgr.free_code(start, stop)
                 real_use -= (stop - start)
                 assert real_use >= 0
             #
             else:
                 # allocate
                 reqsize = random.randrange(1, 200)
-                if random.random() < 0.5:
-                    reqmaxsize = reqsize
-                else:
-                    reqmaxsize = reqsize + random.randrange(0, 200)
-                (start, stop) = self.asmmemmgr.malloc(reqsize, reqmaxsize)
-                assert reqsize <= stop - start < reqmaxsize + 8
+                (start, stop) = self.asmmemmgr.malloc_code(reqsize)
+                assert reqsize <= stop - start < reqsize + 8
                 for otherstart, otherstop in got:           # no overlap
                     assert otherstop <= start or stop <= otherstart
                 got.append((start, stop))
@@ -182,11 +176,11 @@
         assert p[3] == 'y'
         assert p[4] == 'Z'
         assert p[5] == 'z'
-        # 'allblocks' should be one block of length 6 + 15
-        # (15 = alignment - 1) containing the range(rawstart, rawstart + 6)
-        [(blockstart, blockend)] = allblocks
-        assert blockend == blockstart + 6 + (mc.ALIGN_MATERIALIZE - 1)
-        assert blockstart <= rawstart < rawstart + 6 <= blockend
+        # 'allblocks' should be one block of length 16 (= 6 rounded up)
+        # starting at 'rawstart'
+        [blockstart, blockend] = allblocks
+        assert blockend == blockstart + 15  # is odd, one less than real stop
+        assert blockstart == rawstart
         assert puts == [(rawstart + 2, ['a', 'b', 'c', 'd']),
                         (rawstart + 4, ['e', 'f', 'g'])]
 
@@ -232,41 +226,3 @@
 
 def test_blockbuildermixin2():
     test_blockbuildermixin(translated=False)
-
-def test_machinedatablock():
-    ops = []
-    class FakeMemMgr:
-        _addr = 1597
-        def open_malloc(self, minsize):
-            result = (self._addr, self._addr + 100)
-            ops.append(('malloc', minsize) + result)
-            self._addr += 200
-            return result
-        def open_free(self, frm, to):
-            ops.append(('free', frm, to))
-            return to - frm >= 8
-    #
-    allblocks = []
-    md = MachineDataBlockWrapper(FakeMemMgr(), allblocks)
-    p = md.malloc_aligned(26, 16)
-    assert p == 1600
-    assert ops == [('malloc', 26 + 15, 1597, 1697)]
-    del ops[:]
-    #
-    p = md.malloc_aligned(26, 16)
-    assert p == 1632
-    p = md.malloc_aligned(26, 16)
-    assert p == 1664
-    assert allblocks == []
-    assert ops == []
-    #
-    p = md.malloc_aligned(27, 16)
-    assert p == 1808
-    assert allblocks == [(1597, 1697)]
-    assert ops == [('free', 1690, 1697),
-                   ('malloc', 27 + 15, 1797, 1897)]
-    del ops[:]
-    #
-    md.done()
-    assert allblocks == [(1597, 1697), (1797, 1835)]
-    assert ops == [('free', 1835, 1897)]
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -287,6 +287,13 @@
     asmmemmgr_blocks = None
     asmmemmgr_gcreftracers = None
 
+    # asmmemmgr_blocks is a list of integers with the following structure:
+    #  - a pointer, which is an even number
+    #  - optionally an end-pointer minus 1, which is an odd number
+    # If a pointer is followed by a size, together they describe the
+    # start and stop of a piece of code.  In the other case, it is
+    # simply a piece of data that must be free()d.
+
     def __init__(self, cpu, number):
         cpu.tracker.total_compiled_loops += 1
         self.cpu = cpu
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -4,7 +4,6 @@
 
 from rpython.jit.backend.llsupport import symbolic, jitframe, rewrite
 from rpython.jit.backend.llsupport.assembler import (GuardToken, BaseAssembler, debug_bridge)
-from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.metainterp.history import (Const, VOID, ConstInt)
 from rpython.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT
@@ -62,12 +61,14 @@
         self.malloc_slowpath_varsize = 0
         self.wb_slowpath = [0, 0, 0, 0, 0]
         self.setup_failure_recovery()
-        self.datablockwrapper = None
         self.stack_check_slowpath = 0
         self.propagate_exception_path = 0
         self.teardown()
 
     def setup_once(self):
+        # make a list that will be forgotten at the first setup(), for
+        # allocating the few immortal pieces of data now
+        self.allblocks = []
         BaseAssembler.setup_once(self)
         if self.cpu.supports_floats:
             support.ensure_sse2_floats()
@@ -82,13 +83,9 @@
             self.pending_memoryerror_trampoline_from = []
             self.error_trampoline_64 = 0
         self.mc = codebuf.MachineCodeBlockWrapper()
-        #assert self.datablockwrapper is None --- but obscure case
-        # possible, e.g. getting MemoryError and continuing
-        allblocks = self.get_asmmemmgr_blocks(looptoken)
-        self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr,
-                                                        allblocks)
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
+        self.allblocks = self.get_asmmemmgr_blocks(looptoken)
 
     def teardown(self):
         self.pending_guard_tokens = None
@@ -96,6 +93,7 @@
             self.pending_memoryerror_trampoline_from = None
         self.mc = None
         self.current_clt = None
+        self.allblocks = None
 
     def _build_float_constants(self):
         # 0x80000000000000008000000000000000
@@ -111,9 +109,7 @@
         data = neg_const + abs_const + \
                single_neg_const + single_abs_const + \
                zero_const
-        datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, [])
-        float_constants = datablockwrapper.malloc_aligned(len(data), alignment=16)
-        datablockwrapper.done()
+        float_constants = self.malloc_aligned(len(data), alignment=16)
         addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants)
         for i in range(len(data)):
             addr[i] = data[i]
@@ -484,8 +480,7 @@
         if self.cpu.HAS_CODEMAP:
             self.codemap_builder.enter_portal_frame(jd_id, unique_id,
                                                     self.mc.get_relative_pos())
-        frame_info = self.datablockwrapper.malloc_aligned(
-            jitframe.JITFRAMEINFO_SIZE, alignment=WORD)
+        frame_info = self.malloc_aligned(jitframe.JITFRAMEINFO_SIZE)
         clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info)
         clt.frame_info.clear() # for now
 
@@ -647,11 +642,7 @@
         assert isinstance(faildescr, ResumeGuardDescr)
         assert asminfo.rawstart != 0
         self.mc = codebuf.MachineCodeBlockWrapper()
-        allblocks = self.get_asmmemmgr_blocks(looptoken)
-        self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr,
-                                                   allblocks)
-        frame_info = self.datablockwrapper.malloc_aligned(
-            jitframe.JITFRAMEINFO_SIZE, alignment=WORD)
+        frame_info = self.malloc_aligned(jitframe.JITFRAMEINFO_SIZE)
 
         self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         # if accumulation is saved at the guard, we need to update it here!
@@ -717,8 +708,7 @@
         elif IS_X86_32:
             # allocate the gc table right now.  This lets us write
             # machine code with absolute 32-bit addressing.
-            self.gc_table_addr = self.datablockwrapper.malloc_aligned(
-                gcref_table_size, alignment=WORD)
+            self.gc_table_addr = self.malloc_aligned(gcref_table_size)
         #
         self.setup_gcrefs_list(allgcrefs)
 
@@ -854,11 +844,8 @@
         mc.copy_to_raw_memory(adr)
 
     def materialize_loop(self, looptoken):
-        self.datablockwrapper.done()      # finish using cpu.asmmemmgr
-        self.datablockwrapper = None
-        allblocks = self.get_asmmemmgr_blocks(looptoken)
         size = self.mc.get_relative_pos()
-        res = self.mc.materialize(self.cpu, allblocks,
+        res = self.mc.materialize(self.cpu, self.allblocks,
                                   self.cpu.gc_ll_descr.gcrootmap)
         if self.cpu.HAS_CODEMAP:
             self.cpu.codemap.register_codemap(
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -68,13 +68,13 @@
     save_around_call_regs = all_regs
 
     def convert_to_imm(self, c):
-        adr = self.assembler.datablockwrapper.malloc_aligned(8, 8)
+        adr = self.assembler.malloc_aligned(8, 8)
         x = c.getfloatstorage()
         rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0] = x
         return ConstFloatLoc(adr)
 
     def convert_to_imm_16bytes_align(self, c):
-        adr = self.assembler.datablockwrapper.malloc_aligned(16, 16)
+        adr = self.assembler.malloc_aligned(16, 16)
         x = c.getfloatstorage()
         y = longlong.ZEROF
         rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0] = x
@@ -89,14 +89,14 @@
         return loc
 
     def expand_double_float(self, f):
-        adr = self.assembler.datablockwrapper.malloc_aligned(16, 16)
+        adr = self.assembler.malloc_aligned(16, 16)
         fs = f.getfloatstorage()
         rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0] = fs
         rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[1] = fs
         return ConstFloatLoc(adr)
 
     def expand_single_float(self, f):
-        adr = self.assembler.datablockwrapper.malloc_aligned(16, 16)
+        adr = self.assembler.malloc_aligned(16, 16)
         fs = rffi.cast(lltype.SingleFloat, f.getfloatstorage())
         rffi.cast(rffi.CArrayPtr(lltype.SingleFloat), adr)[0] = fs
         rffi.cast(rffi.CArrayPtr(lltype.SingleFloat), adr)[1] = fs

From pypy.commits at gmail.com  Wed Aug 17 11:26:34 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 17 Aug 2016 08:26:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Add lookup for __anext__ in next()
 and __anext__ to typedef of coroutines, add better (offline) async_for test,
 remove old async_for test and change it back to test only the gil error
Message-ID: <57b4822a.c1e31c0a.3cb07.fd35@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86250:d819f16ccf3f
Date: 2016-08-17 17:25 +0200
http://bitbucket.org/pypy/pypy/changeset/d819f16ccf3f/

Log:	Add lookup for __anext__ in next() and __anext__ to typedef of
	coroutines, add better (offline) async_for test, remove old
	async_for test and change it back to test only the gil error

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -801,6 +801,8 @@
     __repr__   = interp2app(Coroutine.descr__repr__),
     __reduce__   = interp2app(Coroutine.descr__reduce__),
     __setstate__ = interp2app(Coroutine.descr__setstate__),
+    __next__   = interp2app(Coroutine.descr_next,
+                            descrmismatch='__anext__'),
     send       = interp2app(Coroutine.descr_send,
                             descrmismatch='send'),
     throw      = interp2app(Coroutine.descr_throw,
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -15,9 +15,6 @@
 
 async def f():
     reader, writer = await asyncio.open_connection('example.com', 80)
-    writer.write(b'a')
-    async for line in reader:
-        print('>>>', line)
     writer.close()
 
 loop = asyncio.get_event_loop()
@@ -25,6 +22,39 @@
 print("done with async loop")
         """
     
+    def test_async_for(self):
+        # temporary test from
+        # http://blog.idego.pl/2015/12/05/back-to-the-future-with-async-and-await-in-python-3-5/
+        """
+import asyncio
+import logging
+import sys
+logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(asctime)s: %(message)s")
+
+class AsyncIter:
+    def __init__(self):
+        self._data = list(range(10))
+        self._index = 0
+    
+    async def __aiter__(self):
+        return self
+    
+    async def __anext__(self):
+        while self._index < 10:
+            await asyncio.sleep(1)
+            self._index += 1
+            return self._data[self._index-1]
+        raise StopAsyncIteration
+
+async def do_loop():
+    async for x in AsyncIter():
+        logging.info(x)
+
+loop = asyncio.get_event_loop()
+futures = [asyncio.ensure_future(do_loop()), asyncio.ensure_future(do_loop())]
+loop.run_until_complete(asyncio.wait(futures))
+        """
+    
     def test_asynchronous_context_managers(self):
         # it is important that "releasing lock A" happens before "holding lock B"
         # or the other way around, but it is not allowed that both coroutines
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -298,7 +298,10 @@
         return w_iter
 
     def next(space, w_obj):
-        w_descr = space.lookup(w_obj, '__next__')
+        if space.type(w_obj).name == 'coroutine':
+            w_descr = space.lookup(w_obj, '__anext__')
+        else:
+            w_descr = space.lookup(w_obj, '__next__')
         if w_descr is None:
             raise oefmt(space.w_TypeError,
                         "'%T' object is not an iterator", w_obj)

From pypy.commits at gmail.com  Wed Aug 17 11:33:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 08:33:30 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: fixes
Message-ID: <57b483ca.2472c20a.3b6d9.fa5b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: asmmemmgr-for-code-only
Changeset: r86251:940d3c45a1c5
Date: 2016-08-17 17:32 +0200
http://bitbucket.org/pypy/pypy/changeset/940d3c45a1c5/

Log:	fixes

diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -275,9 +275,7 @@
         size = self.get_relative_pos()
         align = self.ALIGN_MATERIALIZE
         size = (size + align - 1) & ~(align - 1)   # round up
-        malloced = cpu.asmmemmgr.malloc_code(size)
-        rawstart = malloced[0]
-        rawstop = malloced[1]
+        rawstart, rawstop = cpu.asmmemmgr.malloc_code(size)
         assert (rawstart & (align - 1)) == 0, (
             "malloc_code() not aligned to a multiple of ALIGN_MATERIALIZE")
         assert (rawstart & 1) == 0
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -507,7 +507,7 @@
         self.write_pending_failure_recoveries(regalloc)
         full_size = self.mc.get_relative_pos()
         #
-        rawstart = self.materialize_loop(looptoken)
+        rawstart = self.materialize_loop()
         self.patch_gcref_table(looptoken, rawstart)
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
@@ -588,7 +588,7 @@
         self.write_pending_failure_recoveries(regalloc)
         fullsize = self.mc.get_relative_pos()
         #
-        rawstart = self.materialize_loop(original_loop_token)
+        rawstart = self.materialize_loop()
         self.patch_gcref_table(original_loop_token, rawstart)
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
@@ -642,6 +642,7 @@
         assert isinstance(faildescr, ResumeGuardDescr)
         assert asminfo.rawstart != 0
         self.mc = codebuf.MachineCodeBlockWrapper()
+        self.allblocks = self.get_asmmemmgr_blocks(looptoken)
         frame_info = self.malloc_aligned(jitframe.JITFRAMEINFO_SIZE)
 
         self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
@@ -676,11 +677,12 @@
         self.mc.JMP_l(0)
         self.mc.writeimm32(0)
         self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
-        rawstart = self.materialize_loop(looptoken)
+        rawstart = self.materialize_loop()
         # update the jump (above) to the real trace
         self._patch_jump_to(rawstart + offset, asminfo.rawstart)
         # update the guard to jump right to this custom piece of assembler
         self.patch_jump_for_descr(faildescr, rawstart)
+        self.allblocks = None
 
     def _patch_jump_to(self, adr_jump_offset, adr_new_target):
         assert adr_jump_offset != 0
@@ -843,7 +845,7 @@
         mc.writeimm32(allocated_depth)
         mc.copy_to_raw_memory(adr)
 
-    def materialize_loop(self, looptoken):
+    def materialize_loop(self):
         size = self.mc.get_relative_pos()
         res = self.mc.materialize(self.cpu, self.allblocks,
                                   self.cpu.gc_ll_descr.gcrootmap)
diff --git a/rpython/jit/backend/x86/test/test_x86vector.py b/rpython/jit/backend/x86/test/test_x86vector.py
--- a/rpython/jit/backend/x86/test/test_x86vector.py
+++ b/rpython/jit/backend/x86/test/test_x86vector.py
@@ -31,7 +31,7 @@
 class TestAssembler(BaseTestAssembler):
 
     def imm_4_int32(self, a, b, c, d):
-        adr = self.xrm.assembler.datablockwrapper.malloc_aligned(16, 16)
+        adr = self.xrm.assembler.malloc_aligned(16, 16)
         ptr = rffi.cast(rffi.CArrayPtr(rffi.INT), adr)
         ptr[0] = rffi.r_int(a)
         ptr[1] = rffi.r_int(b)

From pypy.commits at gmail.com  Wed Aug 17 12:01:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 09:01:13 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Test fix, comments
Message-ID: <57b48a49.c4ebc20a.96bf2.f214@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: asmmemmgr-for-code-only
Changeset: r86252:b8f4377bbbb5
Date: 2016-08-17 18:00 +0200
http://bitbucket.org/pypy/pypy/changeset/b8f4377bbbb5/

Log:	Test fix, comments

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -178,6 +178,14 @@
         return clt.asmmemmgr_gcreftracers
 
     def malloc_aligned(self, size, alignment=WORD):
+        """Return a pointer (as an integer) to a malloc()ed piece of
+        data of the given size, with the given alignment (a power of
+        two).  An entry is added to self.allblocks, which should be
+        get_asmmemmgr_blocks(looptoken), so the memory is free()d when
+        the looptoken is freed.  Should be suitable for very small
+        allocations, e.g. two or three WORDs, because the management
+        data saved is only one WORD.
+        """
         p1 = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw',
                            track_allocation=False)
         s1 = s2 = rffi.cast(lltype.Signed, p1)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -290,8 +290,8 @@
     # asmmemmgr_blocks is a list of integers with the following structure:
     #  - a pointer, which is an even number
     #  - optionally an end-pointer minus 1, which is an odd number
-    # If a pointer is followed by a size, together they describe the
-    # start and stop of a piece of code.  In the other case, it is
+    # If a pointer is followed by an end-pointer, together they describe
+    # the start and stop of a piece of code.  In the other case, it is
     # simply a piece of data that must be free()d.
 
     def __init__(self, cpu, number):
diff --git a/rpython/jit/backend/x86/test/test_assembler.py b/rpython/jit/backend/x86/test/test_assembler.py
--- a/rpython/jit/backend/x86/test/test_assembler.py
+++ b/rpython/jit/backend/x86/test/test_assembler.py
@@ -57,7 +57,7 @@
         self.xrm = X86XMMRegisterManager(None, assembler=asm)
         callback(asm)
         asm.mc.RET()
-        rawstart = asm.materialize_loop(looptoken)
+        rawstart = asm.materialize_loop()
         #
         F = ctypes.CFUNCTYPE(ctypes.c_long)
         fn = ctypes.cast(rawstart, F)

From pypy.commits at gmail.com  Wed Aug 17 12:10:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 09:10:43 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Fix more tests
Message-ID: <57b48c83.271ac20a.d0248.fb93@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: asmmemmgr-for-code-only
Changeset: r86253:3342ecf1974e
Date: 2016-08-17 18:09 +0200
http://bitbucket.org/pypy/pypy/changeset/3342ecf1974e/

Log:	Fix more tests

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -190,15 +190,16 @@
                            track_allocation=False)
         s1 = s2 = rffi.cast(lltype.Signed, p1)
         if (s1 & (alignment - 1)) != 0:   # bah, try again
-            lltype.free(p1, flavor='raw')
-            p1 = lltype.malloc(rffi.CCHARP.TO, size + (alignment - 1),
-                               flavor='raw')
+            lltype.free(p1, flavor='raw', track_allocation=False)
+            p1 = lltype.malloc(rffi.CCHARP.TO, size + (alignment - WORD),
+                               flavor='raw', track_allocation=False)
             s1 = s2 = rffi.cast(lltype.Signed, p1)
-            s2 = (s2 + alignment - 1) & ~(alignment - 1)
+            assert s2 & (WORD - 1) == 0   # must return a word-aligned result
+            s2 = (s2 + alignment - WORD) & ~(alignment - WORD)
         assert self.allblocks is not None
         assert (s1 & 1) == 0   # must be even
-        self.allblocks.append(s1)
-        return s2
+        self.allblocks.append(s1)    # s1 is the address to call free() on
+        return s2                    # s2 is the suitably-aligned result
 
     def set_debug(self, v):
         r = self._debug

From pypy.commits at gmail.com  Wed Aug 17 12:49:30 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 17 Aug 2016 09:49:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: kill unneccessary line
Message-ID: <57b4959a.10a81c0a.b1ccd.48ef@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86254:b5b9e11e782d
Date: 2016-08-17 17:39 +0200
http://bitbucket.org/pypy/pypy/changeset/b5b9e11e782d/

Log:	kill unneccessary line

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -221,7 +221,6 @@
         stack_index = 0
         while True:
             current = self
-            number_to_readd = 0
             number_to_readd, attr = self._find_branch_to_move_into(name, index)
             # we found the attributes further up, need to save the
             # previous values of the attributes we passed

From pypy.commits at gmail.com  Wed Aug 17 13:01:53 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 17 Aug 2016 10:01:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Change __next__ to __anext__ in
 coroutine
Message-ID: <57b49881.465d1c0a.5d947.4848@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86255:bc47d0d39227
Date: 2016-08-17 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/bc47d0d39227/

Log:	Change __next__ to __anext__ in coroutine

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -801,7 +801,7 @@
     __repr__   = interp2app(Coroutine.descr__repr__),
     __reduce__   = interp2app(Coroutine.descr__reduce__),
     __setstate__ = interp2app(Coroutine.descr__setstate__),
-    __next__   = interp2app(Coroutine.descr_next,
+    __anext__   = interp2app(Coroutine.descr_next,
                             descrmismatch='__anext__'),
     send       = interp2app(Coroutine.descr_send,
                             descrmismatch='send'),

From pypy.commits at gmail.com  Wed Aug 17 13:05:40 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 17 Aug 2016 10:05:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Add TODO for __anext__ in coroutine
 wrapper
Message-ID: <57b49964.e97ac20a.b412.1197@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86256:0797bd9ecb43
Date: 2016-08-17 19:04 +0200
http://bitbucket.org/pypy/pypy/changeset/0797bd9ecb43/

Log:	Add TODO for __anext__ in coroutine wrapper

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -797,6 +797,8 @@
 )
 assert not GeneratorIterator.typedef.acceptable_as_base_class  # no __new__
 
+# TODO: to have the same distinction as in cpython 3.5, a wrapper typedef with
+# __anext__ has to be created, and __anext__ has to be removed in coroutine
 Coroutine.typedef = TypeDef("coroutine",
     __repr__   = interp2app(Coroutine.descr__repr__),
     __reduce__   = interp2app(Coroutine.descr__reduce__),

From pypy.commits at gmail.com  Wed Aug 17 13:06:43 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 17 Aug 2016 10:06:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Minor change in TODO for __anext__
Message-ID: <57b499a3.c62f1c0a.ea45f.4456@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86257:2ec473a26e63
Date: 2016-08-17 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/2ec473a26e63/

Log:	Minor change in TODO for __anext__

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -797,8 +797,9 @@
 )
 assert not GeneratorIterator.typedef.acceptable_as_base_class  # no __new__
 
-# TODO: to have the same distinction as in cpython 3.5, a wrapper typedef with
-# __anext__ has to be created, and __anext__ has to be removed in coroutine
+# TODO: to have the same distinction (Coroutine | Iterator) as in cpython 3.5,
+# a wrapper typedef with __anext__ has to be created, and __anext__ has to be
+# removed in coroutine
 Coroutine.typedef = TypeDef("coroutine",
     __repr__   = interp2app(Coroutine.descr__repr__),
     __reduce__   = interp2app(Coroutine.descr__reduce__),

From pypy.commits at gmail.com  Wed Aug 17 17:19:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 17 Aug 2016 14:19:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: Refactor the uid_t/gid_t handling in
 rlib.rposix and in interp_posix.py, 
Message-ID: <57b4d4f1.274fc20a.fd5ed.1bcb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86258:a59647bbf2e6
Date: 2016-08-17 23:00 +0200
http://bitbucket.org/pypy/pypy/changeset/a59647bbf2e6/

Log:	Refactor the uid_t/gid_t handling in rlib.rposix and in
	interp_posix.py, based on the clean-up of CPython 2.7.x (see comment
	in interp_posix).

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1703,6 +1703,23 @@
                 "Python int too large for C unsigned short")
         return value
 
+    def c_uid_t_w(self, w_obj):
+        # xxx assumes that uid_t and gid_t are a C unsigned int.
+        # Equivalent to space.c_uint_w(), with the exception that
+        # it also accepts -1 and converts that to UINT_MAX, which
+        # is (uid_t)-1.  And values smaller than -1 raise
+        # OverflowError, not ValueError.
+        try:
+            return self.c_uint_w(w_obj)
+        except OperationError as e:
+            if e.match(self, self.w_ValueError):
+                # ValueError: cannot convert negative integer to unsigned
+                if self.int_w(w_obj) == -1:
+                    return UINT_MAX
+                raise oefmt(self.w_OverflowError,
+                            "user/group id smaller than minimum (-1)")
+            raise
+
     def truncatedint_w(self, w_obj, allow_conversion=True):
         # Like space.gateway_int_w(), but return the integer truncated
         # instead of raising OverflowError.  For obscure cases only.
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -167,6 +167,9 @@
     def visit_c_ushort(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_c_uid_t(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_truncatedint_w(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -294,6 +297,9 @@
     def visit_c_ushort(self, typ):
         self.run_args.append("space.c_ushort_w(%s)" % (self.scopenext(),))
 
+    def visit_c_uid_t(self, typ):
+        self.run_args.append("space.c_uid_t_w(%s)" % (self.scopenext(),))
+
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
@@ -440,6 +446,9 @@
     def visit_c_ushort(self, typ):
         self.unwrap.append("space.c_ushort_w(%s)" % (self.nextarg(),))
 
+    def visit_c_uid_t(self, typ):
+        self.unwrap.append("space.c_uid_t_w(%s)" % (self.nextarg(),))
+
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -4,7 +4,7 @@
 from rpython.rlib import rposix, rposix_stat
 from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_longlong, intmask
+from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint
 from rpython.rlib.unroll import unrolling_iterable
 
 from pypy.interpreter.gateway import unwrap_spec
@@ -20,20 +20,21 @@
 
 c_int = "c_int"
 
-# CPython 2.7 semantics are too messy to follow exactly,
-# e.g. setuid(-2) works on 32-bit but not on 64-bit.  As a result,
-# we decided to just accept any 'int', i.e. any C signed long, and
-# check that they are in range(-2**31, 2**32).  In other words, we
-# accept any number that is either a signed or an unsigned C int.
-c_uid_t = int
-c_gid_t = int
-if sys.maxint == 2147483647:
-    def check_uid_range(space, num):
-        pass
-else:
-    def check_uid_range(space, num):
-        if num < -(1 << 31) or num >= (1 << 32):
-            raise oefmt(space.w_OverflowError, "integer out of range")
+# CPython 2.7 semantics used to be too messy, differing on 32-bit vs
+# 64-bit, but this was cleaned up in recent 2.7.x.  Now, any function
+# taking a uid_t or gid_t accepts numbers in range(-1, 2**32) as an
+# r_uint, with -1 being equivalent to 2**32-1.  Any function that
+# returns a uid_t or gid_t returns either an int or a long, depending
+# on whether it fits or not, but always positive.
+c_uid_t = 'c_uid_t'
+c_gid_t = 'c_uid_t'
+
+def wrap_uid(space, uid):
+    if uid <= r_uint(sys.maxint):
+        return space.wrap(intmask(uid))
+    else:
+        return space.wrap(uid)     # an unsigned number
+wrap_gid = wrap_uid
 
 def fsencode_w(space, w_obj):
     if space.isinstance_w(w_obj, space.w_unicode):
@@ -912,7 +913,7 @@
 
     Return the current process's user id.
     """
-    return space.wrap(os.getuid())
+    return wrap_uid(space, os.getuid())
 
 @unwrap_spec(arg=c_uid_t)
 def setuid(space, arg):
@@ -920,12 +921,10 @@
 
     Set the current process's user id.
     """
-    check_uid_range(space, arg)
     try:
         os.setuid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_uid_t)
 def seteuid(space, arg):
@@ -933,12 +932,10 @@
 
     Set the current process's effective user id.
     """
-    check_uid_range(space, arg)
     try:
         os.seteuid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_gid_t)
 def setgid(space, arg):
@@ -946,12 +943,10 @@
 
     Set the current process's group id.
     """
-    check_uid_range(space, arg)
     try:
         os.setgid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_gid_t)
 def setegid(space, arg):
@@ -959,12 +954,10 @@
 
     Set the current process's effective group id.
     """
-    check_uid_range(space, arg)
     try:
         os.setegid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(path='str0')
 def chroot(space, path):
@@ -983,21 +976,21 @@
 
     Return the current process's group id.
     """
-    return space.wrap(os.getgid())
+    return wrap_gid(space, os.getgid())
 
 def getegid(space):
     """ getegid() -> gid
 
     Return the current process's effective group id.
     """
-    return space.wrap(os.getegid())
+    return wrap_gid(space, os.getegid())
 
 def geteuid(space):
     """ geteuid() -> euid
 
     Return the current process's effective user id.
     """
-    return space.wrap(os.geteuid())
+    return wrap_uid(space, os.geteuid())
 
 def getgroups(space):
     """ getgroups() -> list of group IDs
@@ -1008,7 +1001,7 @@
         list = os.getgroups()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newlist([space.wrap(e) for e in list])
+    return space.newlist([wrap_gid(space, e) for e in list])
 
 def setgroups(space, w_list):
     """ setgroups(list)
@@ -1017,9 +1010,7 @@
     """
     list = []
     for w_gid in space.unpackiterable(w_list):
-        gid = space.int_w(w_gid)
-        check_uid_range(space, gid)
-        list.append(gid)
+        list.append(space.c_uid_t_w(w_gid))
     try:
         os.setgroups(list[:])
     except OSError as e:
@@ -1093,13 +1084,10 @@
 
     Set the current process's real and effective user ids.
     """
-    check_uid_range(space, ruid)
-    check_uid_range(space, euid)
     try:
         os.setreuid(ruid, euid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(rgid=c_gid_t, egid=c_gid_t)
 def setregid(space, rgid, egid):
@@ -1107,13 +1095,10 @@
 
     Set the current process's real and effective group ids.
     """
-    check_uid_range(space, rgid)
-    check_uid_range(space, egid)
     try:
         os.setregid(rgid, egid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(pid=c_int)
 def getsid(space, pid):
@@ -1150,7 +1135,7 @@
         raise wrap_oserror(space, e)
     return space.wrap(pgid)
 
- at unwrap_spec(fd=c_int, pgid=c_gid_t)
+ at unwrap_spec(fd=c_int, pgid=c_int)
 def tcsetpgrp(space, fd, pgid):
     """ tcsetpgrp(fd, pgid)
 
@@ -1170,9 +1155,9 @@
         (ruid, euid, suid) = os.getresuid()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(ruid),
-                           space.wrap(euid),
-                           space.wrap(suid)])
+    return space.newtuple([wrap_uid(space, ruid),
+                           wrap_uid(space, euid),
+                           wrap_uid(space, suid)])
 
 def getresgid(space):
     """ getresgid() -> (rgid, egid, sgid)
@@ -1183,9 +1168,9 @@
         (rgid, egid, sgid) = os.getresgid()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(rgid),
-                           space.wrap(egid),
-                           space.wrap(sgid)])
+    return space.newtuple([wrap_gid(space, rgid),
+                           wrap_gid(space, egid),
+                           wrap_gid(space, sgid)])
 
 @unwrap_spec(ruid=c_uid_t, euid=c_uid_t, suid=c_uid_t)
 def setresuid(space, ruid, euid, suid):
@@ -1284,8 +1269,6 @@
 @unwrap_spec(path='str0', uid=c_uid_t, gid=c_gid_t)
 def chown(space, path, uid, gid):
     """Change the owner and group id of path to the numeric uid and gid."""
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.chown(path, uid, gid)
     except OSError as e:
@@ -1295,8 +1278,6 @@
 def lchown(space, path, uid, gid):
     """Change the owner and group id of path to the numeric uid and gid.
 This function will not follow symbolic links."""
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.lchown(path, uid, gid)
     except OSError as e:
@@ -1307,8 +1288,6 @@
     """Change the owner and group id of the file given by file descriptor
 fd to the numeric uid and gid."""
     fd = space.c_filedescriptor_w(w_fd)
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.fchown(fd, uid, gid)
     except OSError as e:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -621,8 +621,9 @@
     if hasattr(os, 'setuid'):
         def test_os_setuid_error(self):
             os = self.posix
-            raises(OverflowError, os.setuid, -2**31-1)
+            raises(OverflowError, os.setuid, -2)
             raises(OverflowError, os.setuid, 2**32)
+            raises(OSError, os.setuid, -1)
 
     if hasattr(os, 'getgid'):
         def test_os_getgid(self):
@@ -667,8 +668,11 @@
     if hasattr(os, 'setgid'):
         def test_os_setgid_error(self):
             os = self.posix
-            raises(OverflowError, os.setgid, -2**31-1)
+            raises(OverflowError, os.setgid, -2)
             raises(OverflowError, os.setgid, 2**32)
+            raises(OSError, os.setgid, -1)
+            raises(OSError, os.setgid, -1L)
+            raises(OSError, os.setgid, 2**32-1)
 
     if hasattr(os, 'getsid'):
         def test_os_getsid(self):
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -254,6 +254,8 @@
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
     if not _WIN32:
+        UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
+        GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
         CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
 
         TMS = rffi_platform.Struct(
@@ -1453,32 +1455,33 @@
 def setpgid(pid, gid):
     handle_posix_error('setpgid', c_setpgid(pid, gid))
 
-PID_GROUPS_T = rffi.CArrayPtr(rffi.PID_T)
-c_getgroups = external('getgroups', [rffi.INT, PID_GROUPS_T], rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setgroups = external('setgroups', [rffi.SIZE_T, PID_GROUPS_T], rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_initgroups = external('initgroups', [rffi.CCHARP, rffi.PID_T], rffi.INT,
-                        save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    GID_GROUPS_T = rffi.CArrayPtr(GID_T)
+    c_getgroups = external('getgroups', [rffi.INT, GID_GROUPS_T], rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setgroups = external('setgroups', [rffi.SIZE_T, GID_GROUPS_T], rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_initgroups = external('initgroups', [rffi.CCHARP, GID_T], rffi.INT,
+                            save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('getgroups')
 def getgroups():
     n = handle_posix_error('getgroups',
-                           c_getgroups(0, lltype.nullptr(PID_GROUPS_T.TO)))
-    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+                           c_getgroups(0, lltype.nullptr(GID_GROUPS_T.TO)))
+    groups = lltype.malloc(GID_GROUPS_T.TO, n, flavor='raw')
     try:
         n = handle_posix_error('getgroups', c_getgroups(n, groups))
-        return [widen(groups[i]) for i in range(n)]
+        return [widen_gid(groups[i]) for i in range(n)]
     finally:
         lltype.free(groups, flavor='raw')
 
 @replace_os_function('setgroups')
 def setgroups(gids):
     n = len(gids)
-    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+    groups = lltype.malloc(GID_GROUPS_T.TO, n, flavor='raw')
     try:
         for i in range(n):
-            groups[i] = rffi.cast(rffi.PID_T, gids[i])
+            groups[i] = rffi.cast(GID_T, gids[i])
         handle_posix_error('setgroups', c_setgroups(n, groups))
     finally:
         lltype.free(groups, flavor='raw')
@@ -1529,104 +1532,115 @@
 
 #___________________________________________________________________
 
-c_getuid = external('getuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_geteuid = external('geteuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_setuid = external('setuid', [rffi.INT], rffi.INT,
-                    save_err=rffi.RFFI_SAVE_ERRNO)
-c_seteuid = external('seteuid', [rffi.INT], rffi.INT,
-                     save_err=rffi.RFFI_SAVE_ERRNO)
-c_getgid = external('getgid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_getegid = external('getegid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_setgid = external('setgid', [rffi.INT], rffi.INT,
-                    save_err=rffi.RFFI_SAVE_ERRNO)
-c_setegid = external('setegid', [rffi.INT], rffi.INT,
-                     save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    c_getuid = external('getuid', [], UID_T)
+    c_geteuid = external('geteuid', [], UID_T)
+    c_setuid = external('setuid', [UID_T], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+    c_seteuid = external('seteuid', [UID_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    c_getgid = external('getgid', [], GID_T)
+    c_getegid = external('getegid', [], GID_T)
+    c_setgid = external('setgid', [GID_T], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setegid = external('setegid', [GID_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 
- at replace_os_function('getuid')
-def getuid():
-    return handle_posix_error('getuid', c_getuid())
+    def widen_uid(x):
+        return rffi.cast(lltype.Unsigned, x)
+    widen_gid = widen_uid
 
- at replace_os_function('geteuid')
-def geteuid():
-    return handle_posix_error('geteuid', c_geteuid())
+    # NOTE: the resulting type of functions that return a uid/gid is
+    # always Unsigned.  The argument type of functions that take a
+    # uid/gid should also be Unsigned.
 
- at replace_os_function('setuid')
-def setuid(uid):
-    handle_posix_error('setuid', c_setuid(uid))
+    @replace_os_function('getuid')
+    def getuid():
+        return widen_uid(c_getuid())
 
- at replace_os_function('seteuid')
-def seteuid(uid):
-    handle_posix_error('seteuid', c_seteuid(uid))
+    @replace_os_function('geteuid')
+    def geteuid():
+        return widen_uid(c_geteuid())
 
- at replace_os_function('getgid')
-def getgid():
-    return handle_posix_error('getgid', c_getgid())
+    @replace_os_function('setuid')
+    def setuid(uid):
+        handle_posix_error('setuid', c_setuid(uid))
 
- at replace_os_function('getegid')
-def getegid():
-    return handle_posix_error('getegid', c_getegid())
+    @replace_os_function('seteuid')
+    def seteuid(uid):
+        handle_posix_error('seteuid', c_seteuid(uid))
 
- at replace_os_function('setgid')
-def setgid(gid):
-    handle_posix_error('setgid', c_setgid(gid))
+    @replace_os_function('getgid')
+    def getgid():
+        return widen_gid(c_getgid())
 
- at replace_os_function('setegid')
-def setegid(gid):
-    handle_posix_error('setegid', c_setegid(gid))
+    @replace_os_function('getegid')
+    def getegid():
+        return widen_gid(c_getegid())
 
-c_setreuid = external('setreuid', [rffi.INT, rffi.INT], rffi.INT,
-                      save_err=rffi.RFFI_SAVE_ERRNO)
-c_setregid = external('setregid', [rffi.INT, rffi.INT], rffi.INT,
-                      save_err=rffi.RFFI_SAVE_ERRNO)
+    @replace_os_function('setgid')
+    def setgid(gid):
+        handle_posix_error('setgid', c_setgid(gid))
 
- at replace_os_function('setreuid')
-def setreuid(ruid, euid):
-    handle_posix_error('setreuid', c_setreuid(ruid, euid))
+    @replace_os_function('setegid')
+    def setegid(gid):
+        handle_posix_error('setegid', c_setegid(gid))
 
- at replace_os_function('setregid')
-def setregid(rgid, egid):
-    handle_posix_error('setregid', c_setregid(rgid, egid))
+    c_setreuid = external('setreuid', [UID_T, UID_T], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setregid = external('setregid', [GID_T, GID_T], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
-c_getresuid = external('getresuid', [rffi.INTP] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_getresgid = external('getresgid', [rffi.INTP] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setresuid = external('setresuid', [rffi.INT] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setresgid = external('setresgid', [rffi.INT] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
+    @replace_os_function('setreuid')
+    def setreuid(ruid, euid):
+        handle_posix_error('setreuid', c_setreuid(ruid, euid))
 
- at replace_os_function('getresuid')
-def getresuid():
-    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-    try:
-        handle_posix_error('getresuid',
-                           c_getresuid(rffi.ptradd(out, 0),
-                                       rffi.ptradd(out, 1),
-                                       rffi.ptradd(out, 2)))
-        return (widen(out[0]), widen(out[1]), widen(out[2]))
-    finally:
-        lltype.free(out, flavor='raw')
+    @replace_os_function('setregid')
+    def setregid(rgid, egid):
+        handle_posix_error('setregid', c_setregid(rgid, egid))
 
- at replace_os_function('getresgid')
-def getresgid():
-    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-    try:
-        handle_posix_error('getresgid',
-                           c_getresgid(rffi.ptradd(out, 0),
-                                       rffi.ptradd(out, 1),
-                                       rffi.ptradd(out, 2)))
-        return (widen(out[0]), widen(out[1]), widen(out[2]))
-    finally:
-        lltype.free(out, flavor='raw')
+    UID_T_P = lltype.Ptr(lltype.Array(UID_T, hints={'nolength': True}))
+    GID_T_P = lltype.Ptr(lltype.Array(GID_T, hints={'nolength': True}))
+    c_getresuid = external('getresuid', [UID_T_P] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_getresgid = external('getresgid', [GID_T_P] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setresuid = external('setresuid', [UID_T] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setresgid = external('setresgid', [GID_T] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
 
- at replace_os_function('setresuid')
-def setresuid(ruid, euid, suid):
-    handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+    @replace_os_function('getresuid')
+    def getresuid():
+        out = lltype.malloc(UID_T_P.TO, 3, flavor='raw')
+        try:
+            handle_posix_error('getresuid',
+                               c_getresuid(rffi.ptradd(out, 0),
+                                           rffi.ptradd(out, 1),
+                                           rffi.ptradd(out, 2)))
+            return (widen_uid(out[0]), widen_uid(out[1]), widen_uid(out[2]))
+        finally:
+            lltype.free(out, flavor='raw')
 
- at replace_os_function('setresgid')
-def setresgid(rgid, egid, sgid):
-    handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
+    @replace_os_function('getresgid')
+    def getresgid():
+        out = lltype.malloc(GID_T_P.TO, 3, flavor='raw')
+        try:
+            handle_posix_error('getresgid',
+                               c_getresgid(rffi.ptradd(out, 0),
+                                           rffi.ptradd(out, 1),
+                                           rffi.ptradd(out, 2)))
+            return (widen_gid(out[0]), widen_gid(out[1]), widen_gid(out[2]))
+        finally:
+            lltype.free(out, flavor='raw')
+
+    @replace_os_function('setresuid')
+    def setresuid(ruid, euid, suid):
+        handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+
+    @replace_os_function('setresgid')
+    def setresgid(rgid, egid, sgid):
+        handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
 
 #___________________________________________________________________
 

From pypy.commits at gmail.com  Thu Aug 18 03:01:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 00:01:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2369: on FreeBSD/PowerPC,
 "long double" is probably the same as
Message-ID: <57b55d58.85261c0a.57b2c.28e9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86259:559ab6a0936b
Date: 2016-08-18 09:00 +0200
http://bitbucket.org/pypy/pypy/changeset/559ab6a0936b/

Log:	Issue #2369: on FreeBSD/PowerPC, "long double" is probably the same
	as "double", so the JIT tries to look inside this function. Hide the
	function explicitly.

diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -253,6 +253,7 @@
     sandboxsafe=True)
 
 # split here for JIT backends that don't support floats/longlongs/etc.
+ at jit.dont_look_inside
 def is_nonnull_longdouble(cdata):
     return _is_nonnull_longdouble(read_raw_longdouble_data(cdata))
 def is_nonnull_float(cdata, size):

From pypy.commits at gmail.com  Thu Aug 18 04:11:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 01:11:42 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: merge heads
Message-ID: <57b56dbe.0cce1c0a.c8df5.709c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5675:b55c45c4f64f
Date: 2016-08-18 10:11 +0200
http://bitbucket.org/pypy/extradoc/changeset/b55c45c4f64f/

Log:	merge heads

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -5,6 +5,11 @@
 -------
 
 * Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951 (plan_rich)
+  Seems to work, but test suite hangs to verify the CPython tests.
+* tuple indexing for memory view (plan_rich)
+  Comments: Stronly tied to numpy. Hard to implement, because most of the basics are missing (dimensions/strides)
+  We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
+  Matti's opinion on that would be great!
 
 
 Finished

From pypy.commits at gmail.com  Thu Aug 18 04:11:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 01:11:40 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Add another point
Message-ID: <57b56dbc.28eac20a.e7e57.48d2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5674:f2ad0bddce13
Date: 2016-08-18 09:59 +0200
http://bitbucket.org/pypy/extradoc/changeset/f2ad0bddce13/

Log:	Add another point

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -6,13 +6,21 @@
 
 * Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951 (plan_rich)
 
-* At some point, review lib-python/conftest.py to remove the skips
-  due to deadlocks (search for "XXX:")
 
 Finished
 --------
 
 
+Not in any milestone
+--------------------
+
+* At some point, review lib-python/conftest.py to remove the skips
+  due to deadlocks (search for "XXX:")
+
+* collections.py: ``OrderedDict`` should again be a thin wrapper over
+  ``dict``.  The main pain point is ``move_to_end(last=False)``.  See
+  https://mail.python.org/pipermail/python-dev/2016-August/145837.html
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------

From pypy.commits at gmail.com  Thu Aug 18 04:22:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 18 Aug 2016 01:22:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: started to implement tuple
 indexing and finishing cast of memoryview,
 much more changes are needed to implement that
Message-ID: <57b57040.4bc41c0a.8b4c4.3943@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86260:a25cfe739e3f
Date: 2016-08-17 11:08 +0200
http://bitbucket.org/pypy/pypy/changeset/a25cfe739e3f/

Log:	started to implement tuple indexing and finishing cast of
	memoryview, much more changes are needed to implement that

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1428,6 +1428,7 @@
 
     BUF_SIMPLE   = 0x0000
     BUF_WRITABLE = 0x0001
+    BUF_C        = 0x0002
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -12,7 +12,6 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
 from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
 
-
 class W_MemoryView(W_Root):
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
@@ -24,6 +23,7 @@
         self._hash = -1
         self.format = format
         self.itemsize = itemsize
+        self.flags = 0
 
     def buffer_w_ex(self, space, flags):
         self._check_released(space)
@@ -226,8 +226,18 @@
         return size
 
     def descr_cast(self, space, w_format, w_shape=None):
+        self._check_released(space)
+
+        if not space.isinstance_w(w_obj, space.w_unicode):
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: format argument must be a string"))
+
         # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
-        self._check_released(space)
+        #if self.flags & (space.BUF_CONTIG_RO|space.BUF_C) == 0:
+        #    raise OperationError(space.w_TypeError, \
+        #            space.wrap("memoryview: casts are restricted" \
+        #                       " to C-contiguous views"))
+
         fmt = space.str_w(w_format)
         newitemsize = self.get_native_fmtchar(fmt)
         return W_MemoryView(self.buf, fmt, newitemsize)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -165,3 +165,8 @@
 
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
+
+    @py.test.skip("needs numpy ndarray")
+    def test_tuple_indexing(self):
+        content = ndarray(list(range(12)))
+        assert memoryview(content)[0,0] == 0

From pypy.commits at gmail.com  Thu Aug 18 05:19:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 02:19:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: Link to the great issue #2363 in the
 faq entry.
Message-ID: <57b57d9f.8cc51c0a.9c5b7.5cd6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86262:819f6943562d
Date: 2016-08-18 11:18 +0200
http://bitbucket.org/pypy/pypy/changeset/819f6943562d/

Log:	Link to the great issue #2363 in the faq entry.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -356,6 +356,11 @@
 that a C-level traceback is usually of no help at all in PyPy.
 Debugging PyPy can be annoying.
 
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
 In more details:
 
 * First, please give the exact PyPy version, and the OS.

From pypy.commits at gmail.com  Thu Aug 18 05:36:35 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 18 Aug 2016 02:36:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hex method for memoryview + tests
Message-ID: <57b581a3.85261c0a.57b2c.6186@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86263:66f22d46654d
Date: 2016-08-16 11:13 +0200
http://bitbucket.org/pypy/pypy/changeset/66f22d46654d/

Log:	hex method for memoryview + tests

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -491,6 +491,24 @@
         i += 2
     return data
 
+HEXDIGITS = "0123456789abcdef"
+PY_SIZE_T_MAX = 2**(rffi.sizeof(rffi.SIZE_T)*8)-1
+
+def _array_to_hexstring(space, buf):
+    length = buf.getlength()
+    hexstring = StringBuilder(length*2)
+
+    if length > PY_SIZE_T_MAX/2:
+        raise OperationError(space.w_MemoryError)
+
+    for i in range(length):
+        byte = ord(buf.getitem(i))
+        c = (byte >> 4 & 0xf)
+        hexstring.append(HEXDIGITS[c])
+        c = (byte & 0xf)
+        hexstring.append(HEXDIGITS[c])
+
+    return space.wrap(hexstring.build())
 
 class BytearrayDocstrings:
     """bytearray(iterable_of_ints) -> bytearray
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -232,6 +232,11 @@
         newitemsize = self.get_native_fmtchar(fmt)
         return W_MemoryView(self.buf, fmt, newitemsize)
 
+    def descr_hex(self, space):
+        from pypy.objspace.std.bytearrayobject import _array_to_hexstring
+        self._check_released(space)
+        return _array_to_hexstring(space, self.buf)
+
 
 W_MemoryView.typedef = TypeDef(
     "memoryview",
@@ -250,6 +255,7 @@
     __exit__    = interp2app(W_MemoryView.descr_exit),
     __weakref__ = make_weakref_descr(W_MemoryView),
     cast        = interp2app(W_MemoryView.descr_cast),
+    hex         = interp2app(W_MemoryView.descr_hex),
     tobytes     = interp2app(W_MemoryView.descr_tobytes),
     tolist      = interp2app(W_MemoryView.descr_tolist),
     release     = interp2app(W_MemoryView.descr_release),
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -18,6 +18,7 @@
         assert len(w) == 2
         exc = raises(NotImplementedError, "v[0:2:2]")
         assert str(exc.value) == ""
+        exc = raises(TypeError, "memoryview('foobar')")
 
     def test_rw(self):
         data = bytearray(b'abcefg')
@@ -161,3 +162,6 @@
         raises(ValueError, memoryview(b"foobar")._pypy_raw_address)
         a = memoryview(bytearray(b"foobar"))._pypy_raw_address()
         assert a != 0
+
+    def test_hex(self):
+        assert memoryview(b"abc").hex() == u'616263'

From pypy.commits at gmail.com  Thu Aug 18 05:36:38 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 18 Aug 2016 02:36:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hex method for bytes and bytearray + test
Message-ID: <57b581a6.82ddc20a.68c3f.68f1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86264:2da7b655302d
Date: 2016-08-16 11:55 +0200
http://bitbucket.org/pypy/pypy/changeset/2da7b655302d/

Log:	hex method for bytes and bytearray + test

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -441,6 +441,9 @@
     def descr_copy(self, space):
         return self._new(self.data[:])
 
+    def descr_hex(self, space):
+        return _array_to_hexstring(space, self.data, len(self.data), True)
+
 
 # ____________________________________________________________
 # helpers for slow paths, moved out because they contain loops
@@ -494,15 +497,22 @@
 HEXDIGITS = "0123456789abcdef"
 PY_SIZE_T_MAX = 2**(rffi.sizeof(rffi.SIZE_T)*8)-1
 
-def _array_to_hexstring(space, buf):
-    length = buf.getlength()
+ at specialize.arg(3) # raw access
+def _array_to_hexstring(space, buf, len=0, rawaccess=False):
+    if rawaccess:
+        length = len
+    else:
+        length = buf.getlength()
     hexstring = StringBuilder(length*2)
 
     if length > PY_SIZE_T_MAX/2:
         raise OperationError(space.w_MemoryError)
 
     for i in range(length):
-        byte = ord(buf.getitem(i))
+        if rawaccess:
+            byte = ord(buf[i])
+        else:
+            byte = ord(buf.getitem(i))
         c = (byte >> 4 & 0xf)
         hexstring.append(HEXDIGITS[c])
         c = (byte & 0xf)
@@ -944,6 +954,12 @@
         of the specified width.  B is never truncated.
         """
 
+    def hex():
+        """B.hex() -> unicode
+        Return a string object containing two hexadecimal digits
+        for each byte in the instance B.
+        """
+
 
 W_BytearrayObject.typedef = TypeDef(
     "bytearray",
@@ -1093,6 +1109,8 @@
                        doc=BytearrayDocstrings.clear.__doc__),
     copy = interp2app(W_BytearrayObject.descr_copy,
                          doc=BytearrayDocstrings.copy.__doc__),
+    hex = interp2app(W_BytearrayObject.descr_hex,
+                           doc=BytearrayDocstrings.hex.__doc__),
 )
 W_BytearrayObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -394,6 +394,12 @@
         of the specified width. The string S is never truncated.
         """
 
+    def descr_hex(self, space):
+        """S.hex() -> string
+
+        Creates a hexadecimal string of the bytes object
+        """
+
 
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
@@ -648,6 +654,11 @@
     def descr_upper(self, space):
         return W_BytesObject(self._value.upper())
 
+    def descr_hex(self, space):
+        from pypy.objspace.std.bytearrayobject import _array_to_hexstring
+        return _array_to_hexstring(space, self.buffer_w(space, space.BUF_SIMPLE))
+
+
 
 def _create_list_from_bytes(value):
     # need this helper function to allow the jit to look inside and inline
@@ -827,6 +838,7 @@
 
     fromhex = interp2app(W_BytesObject.descr_fromhex, as_classmethod=True),
     maketrans = interp2app(W_BytesObject.descr_maketrans, as_classmethod=True),
+    hex = interp2app(W_BytesObject.descr_hex)
 )
 W_BytesObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -523,3 +523,7 @@
         result = bytearray.maketrans(b'abc', b'xyz')
         assert result == table
         assert type(result) is bytes
+
+    def test_hex(self):
+        assert bytearray(b'santa claus').hex() == "73616e746120636c617573"
+
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -870,3 +870,10 @@
             def __int__(self):
                 return 42
         raises(TypeError, bytes, A())
+
+    def test_hex(self):
+        assert bytes('santa claus', 'ascii').hex() == "73616e746120636c617573"
+        assert bytes([0x73,0x61,0x6e,0x74,0x61,0x20,0x63,0x6c,0x61,0x75,0x73]).hex() == \
+               "73616e746120636c617573"
+        assert bytes(64).hex() == "00"*64
+

From pypy.commits at gmail.com  Thu Aug 18 05:36:39 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 18 Aug 2016 02:36:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: 32bit translation issue, prebuilt long
Message-ID: <57b581a7.c75dc20a.59ed5.690a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86265:5b83e4237db3
Date: 2016-08-16 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/5b83e4237db3/

Log:	32bit translation issue, prebuilt long

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -3,6 +3,7 @@
 from rpython.rlib.objectmodel import (
     import_from_mixin, newlist_hint, resizelist_hint, specialize)
 from rpython.rlib.buffer import Buffer
+from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rstring import StringBuilder, ByteListBuilder
 from rpython.rlib.debug import check_list_of_chars
 from rpython.rtyper.lltypesystem import rffi
@@ -495,7 +496,7 @@
     return data
 
 HEXDIGITS = "0123456789abcdef"
-PY_SIZE_T_MAX = 2**(rffi.sizeof(rffi.SIZE_T)*8)-1
+PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8)-1)
 
 @specialize.arg(3) # raw access
 def _array_to_hexstring(space, buf, len=0, rawaccess=False):

From pypy.commits at gmail.com  Thu Aug 18 05:36:41 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 18 Aug 2016 02:36:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: missing argument to operation error
Message-ID: <57b581a9.08d11c0a.a9e32.84c4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86266:e82a59c7c604
Date: 2016-08-16 12:05 +0200
http://bitbucket.org/pypy/pypy/changeset/e82a59c7c604/

Log:	missing argument to operation error

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -507,7 +507,7 @@
     hexstring = StringBuilder(length*2)
 
     if length > PY_SIZE_T_MAX/2:
-        raise OperationError(space.w_MemoryError)
+        raise OperationError(space.w_MemoryError, space.w_None)
 
     for i in range(length):
         if rawaccess:

From pypy.commits at gmail.com  Thu Aug 18 05:45:01 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 18 Aug 2016 02:45:01 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: a hack: track the source of
 the value that guard_compatible is about
Message-ID: <57b5839d.47cbc20a.d949f.6c1b@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86267:d547e20900d8
Date: 2016-08-08 18:58 +0200
http://bitbucket.org/pypy/pypy/changeset/d547e20900d8/

Log:	a hack: track the source of the value that guard_compatible is about

	(this might well be reverted, but I want to test its performance)

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -35,6 +35,7 @@
         self.jump_target = -1
         self.frozen = False
 
+
     def frozen_copy(self):
         res = CompatibilityCondition(self.known_valid)
         res.conditions = self.conditions[:]
@@ -177,6 +178,48 @@
                     return False
         return True
 
+    def attach_to_descr(self, descr, guard_value_op, optimizer):
+        from rpython.jit.metainterp.resoperation import AbstractResOp
+        assert descr._compatibility_conditions is None
+        descr._compatibility_conditions = self
+        try:
+            descr.failarg_index = guard_value_op.getfailargs().index(
+                    guard_value_op.getarg(0))
+        except ValueError:
+            return # too bad
+        arg = guard_value_op.getarg(0)
+        if not isinstance(arg, AbstractResOp):
+            return
+        if arg.getopnum() not in (rop.GETFIELD_GC_R, rop.GETFIELD_GC_I, rop.GETFIELD_GC_F):
+            return
+        # again, a bit of pattern matching. The trace quite often looks like this:
+        # x = getfield(obj, <fielddescr>)
+        # guard_compatible(x) [x, obj]
+
+        # if this guard fails, we lose the connection between obj and x, which
+        # means that the new bridge will do two things: a guard_compatible on
+        # x, then later do the read again and have a guard_compatible on the
+        # newly read field. This is bad, because one guard_compatible would be
+        # enough. Thus we keep track of this connection, and seed the heapcache
+        # when starting to trace the bridge with that info.
+
+        source_op = arg.getarg(0)
+        try:
+            source_index = guard_value_op.getfailargs().index(source_op)
+        except ValueError:
+            return
+        fielddescr = arg.getdescr()
+        # check whether the same getfield would still yield the same result at
+        # this point in the trace
+        optheap = optimizer.optheap
+        structinfo = optheap.getptrinfo(source_op)
+        cf = optheap.field_cache(fielddescr)
+        field = cf.getfield_from_cache(optheap, structinfo, fielddescr)
+        if field is arg:
+            # yay! we can pass this info on
+            descr.source_failarg_index = source_index
+            descr.source_fielddescr = fielddescr
+
     def repr_of_conditions(self, argrepr="?"):
         return "\n".join([cond.repr(argrepr) for cond in self.conditions])
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1097,6 +1097,10 @@
         # XXX it would be better to patch the guard properly in the backend,
         # but later
         self.fallback_jump_target = 0
+        # the next two attributes are for tracking where the guarded value came
+        # from
+        self.source_failarg_index = -1
+        self.source_fielddescr = None
 
     def find_compatible(self, cpu, ref):
         """ callback for the CPU: given a value ref, it returns:
@@ -1125,13 +1129,21 @@
         # to this descr
         compat_cond = None
         if self.failarg_index != -1:
-            arg = new_loop.inputargs[self.failarg_index]
             firstop = new_loop.operations[0]
+            opindex = 0
+            if self.source_fielddescr:
+                assert firstop.getopnum() == rop.GETFIELD_GC_R
+                assert firstop.getdescr() is self.source_fielddescr
+                arg = firstop
+                opindex = 1
+                firstop = new_loop.operations[1]
+            else:
+                arg = new_loop.inputargs[self.failarg_index]
             if (firstop.getopnum() == rop.GUARD_COMPATIBLE and
                     firstop.getarg(0) is arg):
                 # a guard_compatible about the same box
                 # remove it, it doesn't have to be checked in the bridge
-                del new_loop.operations[0]
+                del new_loop.operations[opindex]
                 newdescr = firstop.getdescr()
                 assert isinstance(newdescr, GuardCompatibleDescr)
                 compat_cond = newdescr._compatibility_conditions
@@ -1150,16 +1162,11 @@
             self.fallback_jump_target = asminfo.asmaddr
         return asminfo
 
-    def make_a_counter_per_value(self, guard_value_op, index):
-        try:
-            self.failarg_index = guard_value_op.getfailargs().index(
-                    guard_value_op.getarg(0))
-        except ValueError:
-            pass # we don't set the failarg_index, too bad
-
+    def make_a_counter_per_value(self, guard_op, index):
+        pass
         # this is not actually enabling the counter_per_value logic,
         # which right now gives bad results with a GUARD_COMPATIBLE
-        #ResumeGuardDescr.make_a_counter_per_value(self, guard_value_op, index)
+        #ResumeGuardDescr.make_a_counter_per_value(self, guard_op, index)
 
     def repr_of_conditions(self, argrepr="?"):
         if self._compatibility_conditions:
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -731,10 +731,11 @@
             op = self._maybe_replace_guard_value(op, descr)
         elif op.getopnum() == rop.GUARD_COMPATIBLE:
             # XXX randomly stuff things into the descr
-            info = self.getptrinfo(op.getarg(0))
-            assert isinstance(descr, compile.GuardCompatibleDescr)
-            if info is not None and info._compatibility_conditions:
-                descr._compatibility_conditions = info._compatibility_conditions
+            #info = self.getptrinfo(op.getarg(0))
+            #assert isinstance(descr, compile.GuardCompatibleDescr)
+            #if info is not None and info._compatibility_conditions:
+            #    info._compatibility_conditions.attach_to_descr(descr, op, self)
+            pass
         return op
 
     def _maybe_replace_guard_value(self, op, descr):
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -209,6 +209,7 @@
         return False
 
     def optimize_GUARD_COMPATIBLE(self, op):
+        from rpython.jit.metainterp.compile import GuardCompatibleDescr
         arg0 = self.get_box_replacement(op.getarg(0))
         if arg0.is_constant():
             # already subject of guard_value
@@ -238,6 +239,10 @@
                 op.getarg(1))
             self.emit_operation(op)
             info.mark_last_guard(self.optimizer)
+            descr = op.getdescr()
+            assert isinstance(descr, GuardCompatibleDescr)
+            info._compatibility_conditions.attach_to_descr(
+                    descr, op, self.optimizer)
 
     def optimize_GUARD_NO_EXCEPTION(self, op):
         if self.last_emitted_operation is REMOVED:
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1047,7 +1047,22 @@
 
 def rebuild_from_resumedata(metainterp, storage, deadframe,
                             virtualizable_info, greenfield_info):
+    from rpython.jit.metainterp.compile import GuardCompatibleDescr
     resumereader = ResumeDataBoxReader(storage, deadframe, metainterp)
+    # this is a gross and terrible HACK
+    # it should be replaced with something significantly more general
+    sourcebox = None
+    if isinstance(storage, GuardCompatibleDescr) and storage.source_fielddescr:
+        assert storage.source_failarg_index >= 0
+        assert storage.failarg_index >= 0
+        sourcebox = resumereader.decode_box(tag(storage.source_failarg_index, TAGBOX), REF)
+        resbox = metainterp.execute_and_record(
+                rop.GETFIELD_GC_R, storage.source_fielddescr, sourcebox)
+        assert resumereader.liveboxes[storage.failarg_index] is None
+        oldbox = resumereader.decode_box(tag(storage.failarg_index, TAGBOX), REF)
+        assert resbox.getref_base() == oldbox.getref_base()
+        resumereader.liveboxes[storage.failarg_index] = resbox
+    # end hack
     boxes = resumereader.consume_vref_and_vable_boxes(virtualizable_info,
                                                       greenfield_info)
     virtualizable_boxes, virtualref_boxes = boxes
diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py
--- a/rpython/jit/metainterp/test/test_compatible.py
+++ b/rpython/jit/metainterp/test/test_compatible.py
@@ -618,7 +618,8 @@
         class Map(object):
             _immutable_fields_ = ['version?']
 
-            def __init__(self):
+            def __init__(self, num):
+                self.num = num
                 self.version = Version()
                 self.dct = {}
 
@@ -642,10 +643,10 @@
                 map = jit.hint(map, promote_compatible=True)
                 return map.lookup_version(name)
 
-        m1 = Map()
+        m1 = Map(1)
         m1.dct['a'] = 1
         m1.dct['b'] = 2
-        m2 = Map()
+        m2 = Map(2)
         m2.dct['a'] = 1
         m2.dct['b'] = 2
         m2.dct['c'] = 5
@@ -656,7 +657,7 @@
         driver = jit.JitDriver(greens = [], reds = ['n', 'res', 'p'])
 
         def f(n, p):
-            res = 0
+            res = p.map.num
             while n > 0:
                 driver.jit_merge_point(n=n, p=p, res=res)
                 res += p.lookup('a')

From pypy.commits at gmail.com  Thu Aug 18 05:45:02 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 18 Aug 2016 02:45:02 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix translation
Message-ID: <57b5839e.a6a5c20a.d74e.6c75@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86268:5169b51b8fab
Date: 2016-08-09 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/5169b51b8fab/

Log:	fix translation

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -354,7 +354,17 @@
 
     @jit.elidable_compatible(quasi_immut_field_name_for_second_arg="version")
     def _type_compares_by_identity(self, version):
-        return self.terminator.w_cls.compares_by_identity()
+        from pypy.objspace.descroperation import object_hash, type_eq
+        assert version is not None
+        # code duplication with W_TypeObject.compares_by_identity
+        # but we can't use that
+        default_hash = object_hash(self.space)
+        my_eq = self._type_lookup_pure('__eq__')
+        overrides_eq = (my_eq and my_eq is not type_eq(self.space))
+        overrides_eq_cmp_or_hash = (overrides_eq or
+                                    self._type_lookup_pure('__cmp__') or
+                                    self._type_lookup_pure('__hash__') is not default_hash)
+        return not overrides_eq_cmp_or_hash
 
 class Terminator(AbstractAttribute):
     _immutable_fields_ = ['w_cls']

From pypy.commits at gmail.com  Thu Aug 18 05:45:04 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 18 Aug 2016 02:45:04 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: guard_compatible was broken by
 b116f09c4e9d: by removing the quasi-immut
Message-ID: <57b583a0.a710c20a.b630c.6cbd@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r86269:2963715f455b
Date: 2016-08-18 11:44 +0200
http://bitbucket.org/pypy/pypy/changeset/2963715f455b/

Log:	guard_compatible was broken by b116f09c4e9d: by removing the quasi-
	immut operations, the guard_compatible optimizer doesn't see them
	early enough. Fix it by making clearing this cache when emmitting a
	guard_compatible

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -163,6 +163,10 @@
         # maps descrs to {index: CacheEntry} dicts
         self.heap_array_cache = {}
 
+        # maps descrs to CacheEntry, contains only descrs that are
+        # quasi-immutable
+        self.quasi_immut_descrs = {}
+
     def reset_keep_likely_virtuals(self):
         # Update only 'head_version', but 'likely_virtual_version' remains
         # at its older value.
@@ -511,7 +515,17 @@
         cache = self.heap_cache.get(fielddescr, None)
         if cache is None:
             cache = self.heap_cache[fielddescr] = CacheEntry(self)
+        self.quasi_immut_descrs[fielddescr] = cache
         if cache.quasiimmut_seen is not None:
             cache.quasiimmut_seen[box] = None
         else:
             cache.quasiimmut_seen = {box: None}
+
+    def invalidate_quasi_immut(self, box):
+        # slow, but should be rare
+        for fielddescr, cache in self.quasi_immut_descrs.iteritems():
+            if cache.quasiimmut_seen:
+                try:
+                    del cache.quasiimmut_seen[box]
+                except KeyError:
+                    pass
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1209,7 +1209,11 @@
             self.metainterp.generate_guard(rop.GUARD_COMPATIBLE, box, [promoted_box],
                                            resumepc=orgpc)
             self.metainterp.heapcache.have_guard_compatible_now(box)
-            # importantly, there is no replace_box here!
+            # it's necessary to not cache quasi-immutable reads on this box
+            # the guard-compatible machinery needs to see the quasi-immutable
+            # read
+            self.metainterp.heapcache.invalidate_quasi_immut(box)
+            # importantly, there is no replace_box here (unlike guard_value)
 
     @arguments("box", "orgpc")
     def opimpl_guard_class(self, box, orgpc):
diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py
--- a/rpython/jit/metainterp/test/test_compatible.py
+++ b/rpython/jit/metainterp/test/test_compatible.py
@@ -622,6 +622,7 @@
                 self.num = num
                 self.version = Version()
                 self.dct = {}
+                self.classdct = {}
 
             def instantiate(self):
                 return Obj(self)
@@ -630,6 +631,10 @@
             def lookup_version(self, version, name):
                 return self.dct.get(name, -1)
 
+            @jit.elidable_compatible(quasi_immut_field_name_for_second_arg='version')
+            def lookup_class(self, version, name):
+                return self.classdct.get(name, -1)
+
         class Version(object):
             pass
 
@@ -641,18 +646,30 @@
                 map = self.map
                 assert isinstance(map, Map)
                 map = jit.hint(map, promote_compatible=True)
-                return map.lookup_version(name)
+                result = map.lookup_version(name)
+                if result == -1:
+                    return map.lookup_class(name)
+                return result
 
         m1 = Map(1)
         m1.dct['a'] = 1
         m1.dct['b'] = 2
+        m1.classdct['d'] = 4
+        m1.classdct['e'] = 5
         m2 = Map(2)
         m2.dct['a'] = 1
         m2.dct['b'] = 2
         m2.dct['c'] = 5
+        m2.classdct['d'] = 4
+        m2.classdct['e'] = 5
+        m2.classdct['f'] = 5
+
+        m3 = Map(3)
+        m3.version = None
 
         p1 = m1.instantiate()
         p2 = m2.instantiate()
+        p3 = m3.instantiate()
 
         driver = jit.JitDriver(greens = [], reds = ['n', 'res', 'p'])
 
@@ -660,15 +677,21 @@
             res = p.map.num
             while n > 0:
                 driver.jit_merge_point(n=n, p=p, res=res)
-                res += p.lookup('a')
-                res += p.lookup('c')
-                res += p.lookup('b')
+                version = p.map.version
+                if version is not None:
+                    res += p.lookup('a')
+                    res += p.lookup('c')
+                    res += p.lookup('b')
+                    res += p.lookup('d')
+                    res += p.lookup('e')
+                    res += p.lookup('f')
                 n -= 1
             return res
 
         def main(x):
             res = f(100, p1)
             res = f(100, p2)
+            res = f(100, p3)
         main(True)
         main(False)
 
diff --git a/rpython/jit/metainterp/test/test_tracingopts.py b/rpython/jit/metainterp/test/test_tracingopts.py
--- a/rpython/jit/metainterp/test/test_tracingopts.py
+++ b/rpython/jit/metainterp/test/test_tracingopts.py
@@ -538,6 +538,34 @@
         self.check_operations_history(quasiimmut_field=1)
 
 
+    def test_heap_caching_quasi_immutable_clear_after_guard_compatible(self):
+        class A:
+            _immutable_fields_ = ['x?']
+        a1 = A()
+        a1.x = 5
+        a2 = A()
+        a2.x = 7
+
+        @jit.elidable
+        def get(n):
+            if n > 0:
+                return a1
+            return a2
+
+        def g(a):
+            return a.x
+
+        def fn(n):
+            a = get(n)
+            res = g(a)
+            jit.hint(a, promote_compatible=True)
+            return res + a.x
+
+        res = self.interp_operations(fn, [7])
+        assert res == 10
+        self.check_operations_history(quasiimmut_field=2)
+
+
     def test_heap_caching_multiple_tuples(self):
         class Gbl(object):
             pass

From pypy.commits at gmail.com  Thu Aug 18 06:02:01 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 18 Aug 2016 03:02:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Change POP_TOP to DUP_TOP in
 async_for, should fix TypeError after execution of 'async for'
Message-ID: <57b58799.4219c20a.c8f99.70f9@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86270:331fd07f62f8
Date: 2016-08-18 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/331fd07f62f8/

Log:	Change POP_TOP to DUP_TOP in async_for, should fix TypeError after
	execution of 'async for'

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -599,17 +599,21 @@
         b_try_cleanup = self.new_block()
         b_after_loop = self.new_block()
         b_after_loop_else = self.new_block()
+        
         self.emit_jump(ops.SETUP_LOOP, b_after_loop)
         self.push_frame_block(F_BLOCK_LOOP, b_try)
+        
         fr.iter.walkabout(self)
         self.emit_op(ops.GET_AITER)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
+        
         self.use_next_block(b_try)
         # This adds another line, so each for iteration can be traced.
         self.lineno_set = False
         self.emit_jump(ops.SETUP_EXCEPT, b_except)
         self.push_frame_block(F_BLOCK_EXCEPT, b_try)
+        
         self.emit_op(ops.GET_ANEXT)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
@@ -617,8 +621,9 @@
         self.emit_op(ops.POP_BLOCK)
         self.pop_frame_block(F_BLOCK_EXCEPT, b_try)
         self.emit_jump(ops.JUMP_FORWARD, b_after_try)
+        
         self.use_next_block(b_except)
-        self.emit_op(ops.POP_TOP)
+        self.emit_op(ops.DUP_TOP)
         self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopAsyncIteration")
         self.emit_op_arg(ops.COMPARE_OP, 10)
         self.emit_jump(ops.POP_JUMP_IF_FALSE, b_try_cleanup, True)

From pypy.commits at gmail.com  Thu Aug 18 06:47:16 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 18 Aug 2016 03:47:16 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: removed semicolon after
 if which made the next statement executed all the time (test did not
 compile .so file because all warnings are treated as errors)
Message-ID: <57b59234.c2a5c20a.44b61.83c7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86271:32b804646e13
Date: 2016-08-18 12:46 +0200
http://bitbucket.org/pypy/pypy/changeset/32b804646e13/

Log:	removed semicolon after if which made the next statement executed
	all the time (test did not compile .so file because all warnings are
	treated as errors)

diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -61,8 +61,9 @@
 PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
 {
     // init may have already been called
-    if (self->arr.arr != NULL);
+    if (self->arr.arr != NULL) {
         deallocate_MyArray(&self->arr);
+    }
 
     int length = 0;
     static char *kwlist[] = {"length", NULL};

From pypy.commits at gmail.com  Thu Aug 18 07:09:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 04:09:17 -0700 (PDT)
Subject: [pypy-commit] buildbot default: detail
Message-ID: <57b5975d.4abf1c0a.f66df.98ea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1014:879f3ec8e19a
Date: 2016-08-18 13:09 +0200
http://bitbucket.org/pypy/buildbot/changeset/879f3ec8e19a/

Log:	detail

diff --git a/bot2/pypybuildbot/summary.py b/bot2/pypybuildbot/summary.py
--- a/bot2/pypybuildbot/summary.py
+++ b/bot2/pypybuildbot/summary.py
@@ -509,6 +509,10 @@
                     extra = {}
                     if failed:
                         extra = {'class': "failSummary failed"}
+                    elif letter in (' ', '.'):
+                        # failure, but letter is " " or ".".
+                        # Replace with "E", which stands out more
+                        letter = 'E'
                     line.append([" ",html.a(letter, href=longrepr_url,
                                             **extra)])
                 else:

From pypy.commits at gmail.com  Thu Aug 18 08:55:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 05:55:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2270 (part): accept and round
 down floats in resource.setrlimit()
Message-ID: <57b5b040.28eac20a.e7e57.b33e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86272:aa0294523cbe
Date: 2016-08-18 14:47 +0200
http://bitbucket.org/pypy/pypy/changeset/aa0294523cbe/

Log:	Issue #2270 (part): accept and round down floats in
	resource.setrlimit()

diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -45,5 +45,8 @@
 
 def test_setrlimit():
     # minimal "does not crash" test
-    x = resource.getrlimit(resource.RLIMIT_CPU)
-    resource.setrlimit(resource.RLIMIT_CPU, x)
+    x, y = resource.getrlimit(resource.RLIMIT_CPU)
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))
+    x += 0.2
+    y += 0.3
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))    # truncated to ints

From pypy.commits at gmail.com  Thu Aug 18 09:54:41 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 18 Aug 2016 06:54:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Change stack effect of POP_EXCEPT
 from -1 to -2, to compensate the depth in 'async for',
 because all except-finally depths are different in cpython. shorten the
 'async for' test case by 5 seconds
Message-ID: <57b5be21.0205c20a.deaff.cbb1@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86273:e51e6872dbf1
Date: 2016-08-18 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/e51e6872dbf1/

Log:	Change stack effect of POP_EXCEPT from -1 to -2, to compensate the
	depth in 'async for', because all except-finally depths are
	different in cpython. shorten the 'async for' test case by 5 seconds

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -606,7 +606,7 @@
     ops.WITH_CLEANUP_FINISH: -1,  # XXX Sometimes more
     ops.LOAD_BUILD_CLASS: 1,
     ops.POP_BLOCK: 0,
-    ops.POP_EXCEPT: -1,
+    ops.POP_EXCEPT: -2,
     ops.END_FINALLY: -4,     # assume always 4: we pretend that SETUP_FINALLY
                              # pushes 4.  In truth, it would only push 1 and
                              # the corresponding END_FINALLY only pops 1.
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -637,13 +637,17 @@
         
         self.use_next_block(b_try_cleanup)
         self.emit_op(ops.END_FINALLY)
+        
         self.use_next_block(b_after_try)
         self.visit_sequence(fr.body)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_try, True)
+        
         self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP
         self.pop_frame_block(F_BLOCK_LOOP, b_try)
+        
         self.use_next_block(b_after_loop)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_end, True)
+        
         self.use_next_block(b_after_loop_else)
         self.visit_sequence(fr.orelse)
         
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -33,14 +33,14 @@
 
 class AsyncIter:
     def __init__(self):
-        self._data = list(range(10))
+        self._data = list(range(5))
         self._index = 0
     
     async def __aiter__(self):
         return self
     
     async def __anext__(self):
-        while self._index < 10:
+        while self._index < 5:
             await asyncio.sleep(1)
             self._index += 1
             return self._data[self._index-1]

From pypy.commits at gmail.com  Thu Aug 18 10:30:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 07:30:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: frame.clear()
Message-ID: <57b5c676.898b1c0a.724cf.ca88@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86274:79296a54da93
Date: 2016-08-18 16:29 +0200
http://bitbucket.org/pypy/pypy/changeset/79296a54da93/

Log:	frame.clear()

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -6,7 +6,14 @@
 from rpython.rlib import jit
 
 
-class GeneratorIterator(W_Root):
+class GeneratorOrCoroutine(W_Root):
+    """XXX: move the common functionality here!"""
+
+    def descr_close(self):
+        raise NotImplementedError
+
+
+class GeneratorIterator(GeneratorOrCoroutine):
     "An iterator created by a generator."
     _immutable_fields_ = ['pycode']
 
@@ -333,7 +340,7 @@
                                       get_printable_location = get_printable_coroutine_location_genentry,
                                       name='coroutineentry')
 
-class Coroutine(W_Root):
+class Coroutine(GeneratorOrCoroutine):
     "A coroutine object."
     _immutable_fields_ = ['pycode']
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -62,6 +62,7 @@
     __metaclass__ = extendabletype
 
     frame_finished_execution = False
+    frame_generator          = None    # for generators/coroutines
     last_instr               = -1
     last_exception           = None
     f_backref                = jit.vref_None
@@ -240,12 +241,16 @@
 
     def run(self):
         """Start this frame's execution."""
-        if self.getcode().co_flags & pycode.CO_COROUTINE:
-            from pypy.interpreter.generator import Coroutine
-            return self.space.wrap(Coroutine(self))
-        elif self.getcode().co_flags & pycode.CO_GENERATOR:
-            from pypy.interpreter.generator import GeneratorIterator
-            return self.space.wrap(GeneratorIterator(self))
+        if self.getcode().co_flags & (pycode.CO_COROUTINE |
+                                      pycode.CO_GENERATOR):
+            if self.getcode().co_flags & pycode.CO_COROUTINE:
+                from pypy.interpreter.generator import Coroutine
+                gen = Coroutine(self)
+            else:
+                from pypy.interpreter.generator import GeneratorIterator
+                gen = GeneratorIterator(self)
+            self.frame_generator = gen
+            return self.space.wrap(gen)
         else:
             return self.execute_frame()
 
@@ -886,6 +891,29 @@
             frame = frame.f_backref()
         return None
 
+    def descr_clear(self, space):
+        # Clears a random subset of the attributes (e.g. some the fast
+        # locals, but not f_locals).  Also clears last_exception, which
+        # is not quite like CPython when it clears f_exc_* (however
+        # there might not be an observable difference).
+        if not self.frame_finished_execution:
+            if self.frame_generator is None or self.frame_generator.running:
+                raise oefmt(space.w_RuntimeError,
+                            "cannot clear an executing frame")
+            # xxx CPython raises the RuntimeWarning "coroutine was never
+            # awaited" in this case too.  Does it make any sense?
+            self.frame_generator.descr_close()
+
+        self.last_exception = None
+        debug = self.getdebug()
+        if debug is not None:
+            debug.w_f_trace = None
+
+        # clear the locals, including the cell/free vars, and the stack
+        for i in range(len(self.locals_cells_stack_w)):
+            self.locals_cells_stack_w[i] = None
+        self.valuestackdepth = 0
+
 # ____________________________________________________________
 
 def get_block_class(opname):
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -545,3 +545,41 @@
         it = yield_raise()
         assert next(it) is KeyError
         assert next(it) is KeyError
+
+    def test_frame_clear(self):
+        import sys, gc, weakref
+        #
+        raises(RuntimeError, sys._getframe().clear)
+        def g():
+            yield 5
+            raises(RuntimeError, sys._getframe().clear)
+            yield 6
+        assert list(g()) == [5, 6]
+        #
+        class A:
+            pass
+        a1 = A(); a1ref = weakref.ref(a1)
+        a2 = A(); a2ref = weakref.ref(a2)
+        seen = []
+        def f():
+            local_a1 = a1
+            for loc in [5, 6, a2]:
+                try:
+                    yield sys._getframe()
+                finally:
+                    seen.append(42)
+                seen.append(43)
+        gen = f()
+        frame = next(gen)
+        a1 = a2 = None
+        gc.collect(); gc.collect()
+        assert a1ref() is not None
+        assert a2ref() is not None
+        assert seen == []
+        frame.clear()
+        assert seen == [42]
+        gc.collect(); gc.collect()
+        assert a1ref() is None, "locals not cleared"
+        assert a2ref() is None, "stack not cleared"
+        #
+        raises(StopIteration, next, gen)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -605,6 +605,7 @@
 PyFrame.typedef = TypeDef('frame',
     __reduce__ = interp2app(PyFrame.descr__reduce__),
     __setstate__ = interp2app(PyFrame.descr__setstate__),
+    clear = interp2app(PyFrame.descr_clear),
     f_builtins = GetSetProperty(PyFrame.fget_f_builtins),
     f_lineno = GetSetProperty(PyFrame.fget_f_lineno, PyFrame.fset_f_lineno),
     f_back = GetSetProperty(PyFrame.fget_f_back),

From pypy.commits at gmail.com  Thu Aug 18 10:38:29 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 18 Aug 2016 07:38:29 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: defining bf_getbuffer to
 get the buffer_test.c going. it dispatches to wrap_getbuffer (maybe not
 what we want)
Message-ID: <57b5c865.4bc41c0a.8b4c4.c6cd@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86275:dbd4120ff2b0
Date: 2016-08-18 16:37 +0200
http://bitbucket.org/pypy/pypy/changeset/dbd4120ff2b0/

Log:	defining bf_getbuffer to get the buffer_test.c going. it dispatches
	to wrap_getbuffer (maybe not what we want)

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -12,6 +12,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+    import pdb; pdb.set_trace()
     raise NotImplementedError
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -22,6 +22,9 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.sys.version import CPYTHON_VERSION
+
+PY3 = CPYTHON_VERSION[0] == 3
 
 # XXX: Also defined in object.h
 Py_LT = 0
@@ -313,6 +316,10 @@
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
 
+    def getformat(self):
+        import pdb; pdb.set_trace()
+        return 'i'
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
@@ -322,6 +329,10 @@
             space.fromcache(State).check_and_raise_exception(always=True)
         return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
 
+def wrap_getbuffer(space, w_self, w_args, func):
+    import pdb; pdb.set_trace()
+    return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
+
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
         func_target = rffi.cast(richcmpfunc, func)
@@ -486,7 +497,8 @@
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
         api_func = slot_tp_getattro.api_func
-
+    elif name == 'tp_as_buffer':
+        raise NotImplementedError
     elif name == 'tp_call':
         call_fn = w_type.getdictvalue(space, '__call__')
         if call_fn is None:
@@ -850,9 +862,16 @@
 slotdefs = eval(slotdefs_str)
 # PyPy addition
 slotdefs += (
-    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    # XXX that might not be what we want!
+    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""),
 )
 
+if not PY3:
+    slotdefs += (
+        TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    )
+
+
 # partial sort to solve some slot conflicts:
 # Number slots before Mapping slots before Sequence slots.
 # These are the only conflicts between __name__ methods
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -774,6 +774,8 @@
             pto.c_tp_setattro = base.c_tp_setattro
         if not pto.c_tp_getattro:
             pto.c_tp_getattro = base.c_tp_getattro
+        if not pto.c_tp_as_buffer:
+            pto.c_tp_as_buffer = base.c_tp_as_buffer
     finally:
         Py_DecRef(space, base_pyo)
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -60,13 +60,13 @@
         raise ValueError("no raw buffer")
 
     def getformat(self):
-        return 'B'
+        raise NotImplementedError
 
     def getitemsize(self):
-        return 1
+        raise NotImplementedError
 
     def getndim(self):
-        return 1
+        raise NotImplementedError
 
     def getshape(self):
         return [self.getlength()]

From pypy.commits at gmail.com  Thu Aug 18 10:46:12 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 Aug 2016 07:46:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix use of space.appexec so that it
 works in py3k -A tests
Message-ID: <57b5ca34.d41a1c0a.7323d.59fc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86276:33d4dcf474ce
Date: 2016-08-18 15:45 +0100
http://bitbucket.org/pypy/pypy/changeset/33d4dcf474ce/

Log:	Fix use of space.appexec so that it works in py3k -A tests

diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,10 +69,21 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        space.appexec([space.wrap(str(tmpdir))], """(path):
+        cls.w_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
             import _cffi_backend     # force it to be initialized
             import sys
             sys.path.insert(0, path)
+            from re_python_pysrc import ffi
+            del sys.path[0]
+            return ffi
+        """)
+        cls.w_sub_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
+            import _cffi_backend     # force it to be initialized
+            import sys
+            sys.path.insert(0, path)
+            from re_py_subsrc import ffi
+            del sys.path[0]
+            return ffi
         """)
 
     def teardown_method(self, meth):
@@ -86,25 +97,25 @@
 
 
     def test_constant_1(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
         assert type(lib.add42) is _cffi_backend.FFI.CData
 
     def test_dlclose(self):
         import _cffi_backend
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
         e = raises(ffi.error, ffi.dlclose, lib)
@@ -115,18 +126,18 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
             assert p.x == 5
@@ -134,13 +145,13 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
-        from re_py_subsrc import ffi
+        ffi = self.sub_ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
         assert ffi.integer_const('k2') == 121212
@@ -153,7 +164,7 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
         p = ffi.addressof(lib, 'globalvar42')
@@ -163,25 +174,25 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == "hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
         assert str(e.value).startswith(

From pypy.commits at gmail.com  Thu Aug 18 10:53:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 07:53:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Uh. This was probably converted from
 "except KeyboardInterrupt, 
Message-ID: <57b5cbf7.c4ebc20a.fb1e3.e27a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86277:5f6febd4256c
Date: 2016-08-18 16:53 +0200
http://bitbucket.org/pypy/pypy/changeset/5f6febd4256c/

Log:	Uh. This was probably converted from "except KeyboardInterrupt,
	RuntimeError:", which was buggy in the first place

diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py
--- a/pypy/tool/pytest/genreportdata.py
+++ b/pypy/tool/pytest/genreportdata.py
@@ -17,7 +17,7 @@
             resultwc = py.path.svnwc(testresultdir)
             print "updating", resultwc
             resultwc.update()
-        except KeyboardInterrupt as RuntimeError:
+        except (KeyboardInterrupt, RuntimeError):
             raise
         except Exception as e: #py.process.ExecutionFailed,e:
             print >> sys.stderr, "Warning: ",e #Subversion update failed"

From pypy.commits at gmail.com  Thu Aug 18 11:04:08 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 Aug 2016 08:04:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57b5ce68.898b1c0a.1c93f.0323@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86278:69541f67e745
Date: 2016-08-18 15:56 +0100
http://bitbucket.org/pypy/pypy/changeset/69541f67e745/

Log:	hg merge default

diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -356,6 +356,11 @@
 that a C-level traceback is usually of no help at all in PyPy.
 Debugging PyPy can be annoying.
 
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
 In more details:
 
 * First, please give the exact PyPy version, and the OS.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1748,6 +1748,23 @@
                 "Python int too large for C unsigned short")
         return value
 
+    def c_uid_t_w(self, w_obj):
+        # xxx assumes that uid_t and gid_t are a C unsigned int.
+        # Equivalent to space.c_uint_w(), with the exception that
+        # it also accepts -1 and converts that to UINT_MAX, which
+        # is (uid_t)-1.  And values smaller than -1 raise
+        # OverflowError, not ValueError.
+        try:
+            return self.c_uint_w(w_obj)
+        except OperationError as e:
+            if e.match(self, self.w_ValueError):
+                # ValueError: cannot convert negative integer to unsigned
+                if self.int_w(w_obj) == -1:
+                    return UINT_MAX
+                raise oefmt(self.w_OverflowError,
+                            "user/group id smaller than minimum (-1)")
+            raise
+
     def truncatedint_w(self, w_obj, allow_conversion=True):
         # Like space.gateway_int_w(), but return the integer truncated
         # instead of raising OverflowError.  For obscure cases only.
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -177,6 +177,9 @@
     def visit_c_ushort(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_c_uid_t(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_truncatedint_w(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -307,6 +310,9 @@
     def visit_c_ushort(self, typ):
         self.run_args.append("space.c_ushort_w(%s)" % (self.scopenext(),))
 
+    def visit_c_uid_t(self, typ):
+        self.run_args.append("space.c_uid_t_w(%s)" % (self.scopenext(),))
+
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
@@ -456,6 +462,9 @@
     def visit_c_ushort(self, typ):
         self.unwrap.append("space.c_ushort_w(%s)" % (self.nextarg(),))
 
+    def visit_c_uid_t(self, typ):
+        self.unwrap.append("space.c_uid_t_w(%s)" % (self.nextarg(),))
+
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -257,6 +257,7 @@
     sandboxsafe=True)
 
 # split here for JIT backends that don't support floats/longlongs/etc.
+ at jit.dont_look_inside
 def is_nonnull_longdouble(cdata):
     return _is_nonnull_longdouble(read_raw_longdouble_data(cdata))
 def is_nonnull_float(cdata, size):
diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,10 +69,21 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        space.appexec([space.wrap(str(tmpdir))], """(path):
+        cls.w_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
             import _cffi_backend     # force it to be initialized
             import sys
             sys.path.insert(0, path)
+            from re_python_pysrc import ffi
+            del sys.path[0]
+            return ffi
+        """)
+        cls.w_sub_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
+            import _cffi_backend     # force it to be initialized
+            import sys
+            sys.path.insert(0, path)
+            from re_py_subsrc import ffi
+            del sys.path[0]
+            return ffi
         """)
 
     def teardown_method(self, meth):
@@ -86,25 +97,25 @@
 
 
     def test_constant_1(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
         assert type(lib.add42) is _cffi_backend.FFI.CData
 
     def test_dlclose(self):
         import _cffi_backend
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
         e = raises(ffi.error, ffi.dlclose, lib)
@@ -115,18 +126,18 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
             assert p.x == 5
@@ -134,13 +145,13 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
-        from re_py_subsrc import ffi
+        ffi = self.sub_ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
         assert ffi.integer_const('k2') == 121212
@@ -153,7 +164,7 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
         p = ffi.addressof(lib, 'globalvar42')
@@ -163,25 +174,25 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == b"hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
         assert str(e.value).startswith(
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
@@ -268,22 +268,26 @@
     min = 0;
     max = haystacksize;
 
-    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
+    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
         if (value < haystack[pos].uniseq) {
-            if (max == pos) break;
-            else max = pos;
+            if (max != pos) {
+                max = pos;
+                continue;
+            }
         }
         else if (value > haystack[pos].uniseq) {
-            if (min == pos) break;
-            else min = pos;
+            if (min != pos) {
+                min = pos;
+                continue;
+            }
         }
-        else
-            break;
+        break;
+    }
 
-        if (value == haystack[pos].uniseq)
-            return haystack[pos].code;
-        else
-            return DBCINV;
+    if (value == haystack[pos].uniseq) {
+        return haystack[pos].code;
+    }
+    return DBCINV;
 }
 #endif
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -11,7 +11,7 @@
 from rpython.rlib import rposix, rposix_stat
 from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_longlong, intmask
+from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -28,20 +28,21 @@
 
 c_int = "c_int"
 
-# CPython 2.7 semantics are too messy to follow exactly,
-# e.g. setuid(-2) works on 32-bit but not on 64-bit.  As a result,
-# we decided to just accept any 'int', i.e. any C signed long, and
-# check that they are in range(-2**31, 2**32).  In other words, we
-# accept any number that is either a signed or an unsigned C int.
-c_uid_t = int
-c_gid_t = int
-if sys.maxint == 2147483647:
-    def check_uid_range(space, num):
-        pass
-else:
-    def check_uid_range(space, num):
-        if num < -(1 << 31) or num >= (1 << 32):
-            raise oefmt(space.w_OverflowError, "integer out of range")
+# CPython 2.7 semantics used to be too messy, differing on 32-bit vs
+# 64-bit, but this was cleaned up in recent 2.7.x.  Now, any function
+# taking a uid_t or gid_t accepts numbers in range(-1, 2**32) as an
+# r_uint, with -1 being equivalent to 2**32-1.  Any function that
+# returns a uid_t or gid_t returns either an int or a long, depending
+# on whether it fits or not, but always positive.
+c_uid_t = 'c_uid_t'
+c_gid_t = 'c_uid_t'
+
+def wrap_uid(space, uid):
+    if uid <= r_uint(sys.maxint):
+        return space.wrap(intmask(uid))
+    else:
+        return space.wrap(uid)     # an unsigned number
+wrap_gid = wrap_uid
 
 class FileEncoder(object):
     is_unicode = True
@@ -1527,7 +1528,7 @@
 
     Return the current process's user id.
     """
-    return space.wrap(os.getuid())
+    return wrap_uid(space, os.getuid())
 
 @unwrap_spec(arg=c_uid_t)
 def setuid(space, arg):
@@ -1535,12 +1536,10 @@
 
     Set the current process's user id.
     """
-    check_uid_range(space, arg)
     try:
         os.setuid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_uid_t)
 def seteuid(space, arg):
@@ -1548,12 +1547,10 @@
 
     Set the current process's effective user id.
     """
-    check_uid_range(space, arg)
     try:
         os.seteuid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_gid_t)
 def setgid(space, arg):
@@ -1561,12 +1558,10 @@
 
     Set the current process's group id.
     """
-    check_uid_range(space, arg)
     try:
         os.setgid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_gid_t)
 def setegid(space, arg):
@@ -1574,12 +1569,10 @@
 
     Set the current process's effective group id.
     """
-    check_uid_range(space, arg)
     try:
         os.setegid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(path='fsencode')
 def chroot(space, path):
@@ -1598,21 +1591,21 @@
 
     Return the current process's group id.
     """
-    return space.wrap(os.getgid())
+    return wrap_gid(space, os.getgid())
 
 def getegid(space):
     """ getegid() -> gid
 
     Return the current process's effective group id.
     """
-    return space.wrap(os.getegid())
+    return wrap_gid(space, os.getegid())
 
 def geteuid(space):
     """ geteuid() -> euid
 
     Return the current process's effective user id.
     """
-    return space.wrap(os.geteuid())
+    return wrap_uid(space, os.geteuid())
 
 def getgroups(space):
     """ getgroups() -> list of group IDs
@@ -1623,7 +1616,7 @@
         list = os.getgroups()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newlist([space.wrap(e) for e in list])
+    return space.newlist([wrap_gid(space, e) for e in list])
 
 def setgroups(space, w_list):
     """ setgroups(list)
@@ -1632,9 +1625,7 @@
     """
     list = []
     for w_gid in space.unpackiterable(w_list):
-        gid = space.int_w(w_gid)
-        check_uid_range(space, gid)
-        list.append(gid)
+        list.append(space.c_uid_t_w(w_gid))
     try:
         os.setgroups(list[:])
     except OSError as e:
@@ -1708,13 +1699,10 @@
 
     Set the current process's real and effective user ids.
     """
-    check_uid_range(space, ruid)
-    check_uid_range(space, euid)
     try:
         os.setreuid(ruid, euid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(rgid=c_gid_t, egid=c_gid_t)
 def setregid(space, rgid, egid):
@@ -1722,13 +1710,10 @@
 
     Set the current process's real and effective group ids.
     """
-    check_uid_range(space, rgid)
-    check_uid_range(space, egid)
     try:
         os.setregid(rgid, egid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(pid=c_int)
 def getsid(space, pid):
@@ -1785,9 +1770,9 @@
         (ruid, euid, suid) = os.getresuid()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(ruid),
-                           space.wrap(euid),
-                           space.wrap(suid)])
+    return space.newtuple([wrap_uid(space, ruid),
+                           wrap_uid(space, euid),
+                           wrap_uid(space, suid)])
 
 def getresgid(space):
     """ getresgid() -> (rgid, egid, sgid)
@@ -1798,9 +1783,9 @@
         (rgid, egid, sgid) = os.getresgid()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(rgid),
-                           space.wrap(egid),
-                           space.wrap(sgid)])
+    return space.newtuple([wrap_gid(space, rgid),
+                           wrap_gid(space, egid),
+                           wrap_gid(space, sgid)])
 
 @unwrap_spec(ruid=c_uid_t, euid=c_uid_t, suid=c_uid_t)
 def setresuid(space, ruid, euid, suid):
@@ -1922,8 +1907,6 @@
   an open file descriptor.
 dir_fd and follow_symlinks may not be implemented on your platform.
   If they are unavailable, using them will raise a NotImplementedError."""
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     if not (rposix.HAVE_LCHOWN or rposix.HAVE_FCHMODAT):
         if not follow_symlinks:
             raise argument_unavailable(space, 'chown', 'follow_symlinks')
@@ -1969,8 +1952,6 @@
 Change the owner and group id of path to the numeric uid and gid.
 This function will not follow symbolic links.
 Equivalent to os.chown(path, uid, gid, follow_symlinks=False)."""
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.lchown(path, uid, gid)
     except OSError as e:
@@ -1983,8 +1964,6 @@
 Change the owner and group id of the file given by file descriptor
 fd to the numeric uid and gid.  Equivalent to os.chown(fd, uid, gid)."""
     fd = space.c_filedescriptor_w(w_fd)
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.fchown(fd, uid, gid)
     except OSError as e:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -595,8 +595,9 @@
     if hasattr(os, 'setuid'):
         def test_os_setuid_error(self):
             os = self.posix
-            raises(OverflowError, os.setuid, -2**31-1)
+            raises(OverflowError, os.setuid, -2)
             raises(OverflowError, os.setuid, 2**32)
+            raises(OSError, os.setuid, -1)
 
     if hasattr(os, 'getgid'):
         def test_os_getgid(self):
@@ -641,8 +642,10 @@
     if hasattr(os, 'setgid'):
         def test_os_setgid_error(self):
             os = self.posix
-            raises(OverflowError, os.setgid, -2**31-1)
+            raises(OverflowError, os.setgid, -2)
             raises(OverflowError, os.setgid, 2**32)
+            raises(OSError, os.setgid, -1)
+            raises(OSError, os.setgid, 2**32-1)
 
     if hasattr(os, 'getsid'):
         def test_os_getsid(self):
diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -45,5 +45,8 @@
 
 def test_setrlimit():
     # minimal "does not crash" test
-    x = resource.getrlimit(resource.RLIMIT_CPU)
-    resource.setrlimit(resource.RLIMIT_CPU, x)
+    x, y = resource.getrlimit(resource.RLIMIT_CPU)
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))
+    x += 0.2
+    y += 0.3
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))    # truncated to ints
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -221,7 +221,6 @@
         stack_index = 0
         while True:
             current = self
-            number_to_readd = 0
             number_to_readd, attr = self._find_branch_to_move_into(name, index)
             # we found the attributes further up, need to save the
             # previous values of the attributes we passed
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1051,8 +1051,9 @@
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     #
+    jd_name = jitdriver_sd.jitdriver.name
     metainterp_sd.jitlog.start_new_trace(metainterp_sd,
-            faildescr=resumekey, entry_bridge=False)
+            faildescr=resumekey, entry_bridge=False, jd_name=jd_name)
     #
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -98,6 +98,7 @@
     def log_abort_loop(self, trace, memo=None):
         debug_start("jit-abort-log")
         if not have_debug_prints():
+            debug_stop("jit-abort-log")
             return
         inputargs, operations = self._unpack_trace(trace)
         logops = self._log_operations(inputargs, operations, ops_offset=None,
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -64,6 +64,10 @@
     testself.all_graphs = graphs
     result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0]
 
+
+    class FakeJitDriver:
+        name = 'fakejitdriver'
+
     class FakeJitDriverSD:
         num_green_args = 0
         portal_graph = graphs[0]
@@ -72,6 +76,7 @@
         result_type = result_kind
         portal_runner_ptr = "???"
         vec = False
+        jitdriver = FakeJitDriver()
 
     stats = history.Stats(None)
     cpu = CPUClass(rtyper, stats, None, False)
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 1
+JITLOG_VERSION = 2
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -331,7 +331,7 @@
     def finish(self):
         jitlog_teardown()
 
-    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False):
+    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False, jd_name=""):
         # even if the logger is not enabled, increment the trace id
         self.trace_id += 1
         if not jitlog_enabled():
@@ -345,6 +345,7 @@
         else:
             content.append(encode_str('loop'))
             content.append(encode_le_addr(int(entry_bridge)))
+        content.append(encode_str(jd_name))
         self._write_marked(MARK_START_TRACE, ''.join(content))
 
     def trace_aborted(self):
@@ -508,7 +509,9 @@
         """ an operation is written as follows:
             <marker> <opid (16 bit)> \
                      <len (32 bit)> \
-                     <res_val>,<arg_0>,...,<arg_n>,<descr>
+                     <res_val>,<arg_0>,...,<arg_n> \
+                     <descr>
+                     <failarg_0>,...<failarg_n>
             The marker indicates if the last argument is
             a descr or a normal argument.
         """
@@ -517,16 +520,21 @@
         le_opnum = encode_le_16bit(op.getopnum())
         str_res = self.var_to_str(op)
         line = ','.join([str_res] + str_args)
+        failargslist = op.getfailargs()
+        failargs = ''
+        if failargslist:
+            failargs = ','.join([self.var_to_str(farg) for farg in failargslist])
+        #
         if descr:
             descr_str = descr.repr_of_descr()
             line = line + ',' + descr_str
             string = encode_str(line)
             descr_number = compute_unique_id(descr)
             le_descr_number = encode_le_addr(descr_number)
-            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number
+            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number + encode_str(failargs)
         else:
             string = encode_str(line)
-            return MARK_RESOP, le_opnum + string
+            return MARK_RESOP, le_opnum + string + encode_str(failargs)
 
 
     def write_core_dump(self, operations, i, op, ops_offset):
@@ -578,6 +586,8 @@
         return ''.join(dump)
 
     def var_to_str(self, arg):
+        if arg is None:
+            return '-'
         try:
             mv = self.memo[arg]
         except KeyError:
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -48,7 +48,7 @@
         file.ensure()
         fd = file.open('wb')
         jl.jitlog_init(fd.fileno())
-        logger.start_new_trace(self.make_metainterp_sd())
+        logger.start_new_trace(self.make_metainterp_sd(), jd_name='jdname')
         log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
         op = ResOperation(rop.DEBUG_MERGE_POINT, [ConstInt(0), ConstInt(0), ConstInt(0)])
         log_trace.write([], [op])
@@ -58,6 +58,7 @@
         is_32bit = chr(sys.maxint == 2**31-1)
         assert binary == (jl.MARK_START_TRACE) + jl.encode_le_addr(1) + \
                          jl.encode_str('loop') + jl.encode_le_addr(0) + \
+                         jl.encode_str('jdname') + \
                          (jl.MARK_TRACE) + jl.encode_le_addr(1) + \
                          (jl.MARK_INPUT_ARGS) + jl.encode_str('') + \
                          (jl.MARK_INIT_MERGE_POINT) + b'\x05\x00\x01s\x00i\x08s\x00i\x10s' + \
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -96,12 +96,15 @@
             return 0;
         }
     ''',]
+    post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);']
 else:
     separate_module_sources = []
+    post_include_bits = []
     includes=['errno.h','stdio.h']
 errno_eci = ExternalCompilationInfo(
     includes=includes,
     separate_module_sources=separate_module_sources,
+    post_include_bits=post_include_bits,
 )
 
 # Direct getters/setters, don't use directly!
@@ -251,6 +254,8 @@
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
     if not _WIN32:
+        UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
+        GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
         CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
 
         TMS = rffi_platform.Struct(
@@ -1450,32 +1455,33 @@
 def setpgid(pid, gid):
     handle_posix_error('setpgid', c_setpgid(pid, gid))
 
-PID_GROUPS_T = rffi.CArrayPtr(rffi.PID_T)
-c_getgroups = external('getgroups', [rffi.INT, PID_GROUPS_T], rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setgroups = external('setgroups', [rffi.SIZE_T, PID_GROUPS_T], rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_initgroups = external('initgroups', [rffi.CCHARP, rffi.PID_T], rffi.INT,
-                        save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    GID_GROUPS_T = rffi.CArrayPtr(GID_T)
+    c_getgroups = external('getgroups', [rffi.INT, GID_GROUPS_T], rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setgroups = external('setgroups', [rffi.SIZE_T, GID_GROUPS_T], rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_initgroups = external('initgroups', [rffi.CCHARP, GID_T], rffi.INT,
+                            save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('getgroups')
 def getgroups():
     n = handle_posix_error('getgroups',
-                           c_getgroups(0, lltype.nullptr(PID_GROUPS_T.TO)))
-    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+                           c_getgroups(0, lltype.nullptr(GID_GROUPS_T.TO)))
+    groups = lltype.malloc(GID_GROUPS_T.TO, n, flavor='raw')
     try:
         n = handle_posix_error('getgroups', c_getgroups(n, groups))
-        return [widen(groups[i]) for i in range(n)]
+        return [widen_gid(groups[i]) for i in range(n)]
     finally:
         lltype.free(groups, flavor='raw')
 
 @replace_os_function('setgroups')
 def setgroups(gids):
     n = len(gids)
-    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+    groups = lltype.malloc(GID_GROUPS_T.TO, n, flavor='raw')
     try:
         for i in range(n):
-            groups[i] = rffi.cast(rffi.PID_T, gids[i])
+            groups[i] = rffi.cast(GID_T, gids[i])
         handle_posix_error('setgroups', c_setgroups(n, groups))
     finally:
         lltype.free(groups, flavor='raw')
@@ -1526,104 +1532,115 @@
 
 #___________________________________________________________________
 
-c_getuid = external('getuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_geteuid = external('geteuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_setuid = external('setuid', [rffi.INT], rffi.INT,
-                    save_err=rffi.RFFI_SAVE_ERRNO)
-c_seteuid = external('seteuid', [rffi.INT], rffi.INT,
-                     save_err=rffi.RFFI_SAVE_ERRNO)
-c_getgid = external('getgid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_getegid = external('getegid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_setgid = external('setgid', [rffi.INT], rffi.INT,
-                    save_err=rffi.RFFI_SAVE_ERRNO)
-c_setegid = external('setegid', [rffi.INT], rffi.INT,
-                     save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    c_getuid = external('getuid', [], UID_T)
+    c_geteuid = external('geteuid', [], UID_T)
+    c_setuid = external('setuid', [UID_T], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+    c_seteuid = external('seteuid', [UID_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    c_getgid = external('getgid', [], GID_T)
+    c_getegid = external('getegid', [], GID_T)
+    c_setgid = external('setgid', [GID_T], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setegid = external('setegid', [GID_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 
- at replace_os_function('getuid')
-def getuid():
-    return handle_posix_error('getuid', c_getuid())
+    def widen_uid(x):
+        return rffi.cast(lltype.Unsigned, x)
+    widen_gid = widen_uid
 
- at replace_os_function('geteuid')
-def geteuid():
-    return handle_posix_error('geteuid', c_geteuid())
+    # NOTE: the resulting type of functions that return a uid/gid is
+    # always Unsigned.  The argument type of functions that take a
+    # uid/gid should also be Unsigned.
 
- at replace_os_function('setuid')
-def setuid(uid):
-    handle_posix_error('setuid', c_setuid(uid))
+    @replace_os_function('getuid')
+    def getuid():
+        return widen_uid(c_getuid())
 
- at replace_os_function('seteuid')
-def seteuid(uid):
-    handle_posix_error('seteuid', c_seteuid(uid))
+    @replace_os_function('geteuid')
+    def geteuid():
+        return widen_uid(c_geteuid())
 
- at replace_os_function('getgid')
-def getgid():
-    return handle_posix_error('getgid', c_getgid())
+    @replace_os_function('setuid')
+    def setuid(uid):
+        handle_posix_error('setuid', c_setuid(uid))
 
- at replace_os_function('getegid')
-def getegid():
-    return handle_posix_error('getegid', c_getegid())
+    @replace_os_function('seteuid')
+    def seteuid(uid):
+        handle_posix_error('seteuid', c_seteuid(uid))
 
- at replace_os_function('setgid')
-def setgid(gid):
-    handle_posix_error('setgid', c_setgid(gid))
+    @replace_os_function('getgid')
+    def getgid():
+        return widen_gid(c_getgid())
 
- at replace_os_function('setegid')
-def setegid(gid):
-    handle_posix_error('setegid', c_setegid(gid))
+    @replace_os_function('getegid')
+    def getegid():
+        return widen_gid(c_getegid())
 
-c_setreuid = external('setreuid', [rffi.INT, rffi.INT], rffi.INT,
-                      save_err=rffi.RFFI_SAVE_ERRNO)
-c_setregid = external('setregid', [rffi.INT, rffi.INT], rffi.INT,
-                      save_err=rffi.RFFI_SAVE_ERRNO)
+    @replace_os_function('setgid')
+    def setgid(gid):
+        handle_posix_error('setgid', c_setgid(gid))
 
- at replace_os_function('setreuid')
-def setreuid(ruid, euid):
-    handle_posix_error('setreuid', c_setreuid(ruid, euid))
+    @replace_os_function('setegid')
+    def setegid(gid):
+        handle_posix_error('setegid', c_setegid(gid))
 
- at replace_os_function('setregid')
-def setregid(rgid, egid):
-    handle_posix_error('setregid', c_setregid(rgid, egid))
+    c_setreuid = external('setreuid', [UID_T, UID_T], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setregid = external('setregid', [GID_T, GID_T], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
-c_getresuid = external('getresuid', [rffi.INTP] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_getresgid = external('getresgid', [rffi.INTP] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setresuid = external('setresuid', [rffi.INT] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setresgid = external('setresgid', [rffi.INT] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
+    @replace_os_function('setreuid')
+    def setreuid(ruid, euid):
+        handle_posix_error('setreuid', c_setreuid(ruid, euid))
 
- at replace_os_function('getresuid')
-def getresuid():
-    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-    try:
-        handle_posix_error('getresuid',
-                           c_getresuid(rffi.ptradd(out, 0),
-                                       rffi.ptradd(out, 1),
-                                       rffi.ptradd(out, 2)))
-        return (widen(out[0]), widen(out[1]), widen(out[2]))
-    finally:
-        lltype.free(out, flavor='raw')
+    @replace_os_function('setregid')
+    def setregid(rgid, egid):
+        handle_posix_error('setregid', c_setregid(rgid, egid))
 
- at replace_os_function('getresgid')
-def getresgid():
-    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-    try:
-        handle_posix_error('getresgid',
-                           c_getresgid(rffi.ptradd(out, 0),
-                                       rffi.ptradd(out, 1),
-                                       rffi.ptradd(out, 2)))
-        return (widen(out[0]), widen(out[1]), widen(out[2]))
-    finally:
-        lltype.free(out, flavor='raw')
+    UID_T_P = lltype.Ptr(lltype.Array(UID_T, hints={'nolength': True}))
+    GID_T_P = lltype.Ptr(lltype.Array(GID_T, hints={'nolength': True}))
+    c_getresuid = external('getresuid', [UID_T_P] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_getresgid = external('getresgid', [GID_T_P] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setresuid = external('setresuid', [UID_T] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setresgid = external('setresgid', [GID_T] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
 
- at replace_os_function('setresuid')
-def setresuid(ruid, euid, suid):
-    handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+    @replace_os_function('getresuid')
+    def getresuid():
+        out = lltype.malloc(UID_T_P.TO, 3, flavor='raw')
+        try:
+            handle_posix_error('getresuid',
+                               c_getresuid(rffi.ptradd(out, 0),
+                                           rffi.ptradd(out, 1),
+                                           rffi.ptradd(out, 2)))
+            return (widen_uid(out[0]), widen_uid(out[1]), widen_uid(out[2]))
+        finally:
+            lltype.free(out, flavor='raw')
 
- at replace_os_function('setresgid')
-def setresgid(rgid, egid, sgid):
-    handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
+    @replace_os_function('getresgid')
+    def getresgid():
+        out = lltype.malloc(GID_T_P.TO, 3, flavor='raw')
+        try:
+            handle_posix_error('getresgid',
+                               c_getresgid(rffi.ptradd(out, 0),
+                                           rffi.ptradd(out, 1),
+                                           rffi.ptradd(out, 2)))
+            return (widen_gid(out[0]), widen_gid(out[1]), widen_gid(out[2]))
+        finally:
+            lltype.free(out, flavor='raw')
+
+    @replace_os_function('setresuid')
+    def setresuid(ruid, euid, suid):
+        handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+
+    @replace_os_function('setresgid')
+    def setresgid(rgid, egid, sgid):
+        handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
 
 #___________________________________________________________________
 
@@ -2046,3 +2063,40 @@
     def mknodat(path, mode, device, dir_fd=AT_FDCWD):
         error = c_mknodat(dir_fd, path, mode, device)
         handle_posix_error('mknodat', error)
+
+
+eci_inheritable = eci.merge(ExternalCompilationInfo(
+    separate_module_sources=["""
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable)
+{
+    /* XXX minimal impl. XXX */
+    int request = inheritable ? FIONCLEX : FIOCLEX;
+    return ioctl(fd, request, NULL);
+}
+RPY_EXTERN
+int rpy_get_inheritable(int fd)
+{
+    int flags = fcntl(fd, F_GETFD, 0);
+    if (flags == -1)
+        return -1;
+    return !(flags & FD_CLOEXEC);
+}
+    """],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+
+def set_inheritable(fd, inheritable):
+    error = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', error)
+
+def get_inheritable(fd):
+    res = c_get_inheritable(fd)
+    res = handle_posix_error('get_inheritable', res)
+    return res != 0
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -572,3 +572,12 @@
         os.close(dirfd)
     assert tmpdir.join('file').check(exists=False)
     assert tmpdir.join('file2').check(exists=True)
+
+def test_set_inheritable():
+    fd1, fd2 = os.pipe()
+    rposix.set_inheritable(fd1, True)
+    assert rposix.get_inheritable(fd1) == True
+    rposix.set_inheritable(fd1, False)
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)

From pypy.commits at gmail.com  Thu Aug 18 11:04:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 08:04:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add the new RecursionError exception.
Message-ID: <57b5ce78.85261c0a.57b2c.dc73@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86279:044728302224
Date: 2016-08-18 17:03 +0200
http://bitbucket.org/pypy/pypy/changeset/044728302224/

Log:	Add the new RecursionError exception.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1936,6 +1936,7 @@
     'PendingDeprecationWarning',
     'ReferenceError',
     'ResourceWarning',
+    'RecursionError',
     'RuntimeError',
     'RuntimeWarning',
     'StopIteration',
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -720,7 +720,7 @@
             raise OperationError(space.w_MemoryError, space.w_None)
         except rstackovf.StackOverflow as e:
             rstackovf.check_stack_overflow()
-            raise oefmt(space.w_RuntimeError,
+            raise oefmt(space.w_RecursionError,
                         "maximum recursion depth exceeded")
         except RuntimeError:   # not on top of py.py
             raise OperationError(space.w_RuntimeError, space.w_None)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -96,7 +96,7 @@
             # Note that this case catches AttributeError!
             rstackovf.check_stack_overflow()
             next_instr = self.handle_asynchronous_error(ec,
-                self.space.w_RuntimeError,
+                self.space.w_RecursionError,
                 self.space.wrap("maximum recursion depth exceeded"))
         return next_instr
 
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -407,7 +407,7 @@
         def f(): f()
         try:
             f()
-        except RuntimeError as e:
+        except RecursionError as e:
             assert str(e) == "maximum recursion depth exceeded"
         else:
             assert 0, "should have raised!"
diff --git a/pypy/module/exceptions/__init__.py b/pypy/module/exceptions/__init__.py
--- a/pypy/module/exceptions/__init__.py
+++ b/pypy/module/exceptions/__init__.py
@@ -47,6 +47,7 @@
         'PendingDeprecationWarning' : 'interp_exceptions.W_PendingDeprecationWarning',
         'PermissionError': 'interp_exceptions.W_PermissionError',
         'ProcessLookupError': 'interp_exceptions.W_ProcessLookupError',
+        'RecursionError' : 'interp_exceptions.W_RecursionError',
         'ReferenceError' : 'interp_exceptions.W_ReferenceError',
         'ResourceWarning'  : 'interp_exceptions.W_ResourceWarning',
         'RuntimeError' : 'interp_exceptions.W_RuntimeError',
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -64,6 +64,7 @@
       +-- ReferenceError
       +-- RuntimeError
       |    +-- NotImplementedError
+      |    +-- RecursionError
       +-- SyntaxError
       |    +-- IndentationError
       |         +-- TabError
@@ -921,6 +922,9 @@
 W_NotImplementedError = _new_exception('NotImplementedError', W_RuntimeError,
                         """Method or function hasn't been implemented yet.""")
 
+W_RecursionError = _new_exception('RecursionError', W_RuntimeError,
+                        """Recursion limit exceeded.""")
+
 W_AttributeError = _new_exception('AttributeError', W_Exception,
                                   """Attribute not found.""")
 
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -44,8 +44,8 @@
 @unwrap_spec(new_limit="c_int")
 def setrecursionlimit(space, new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
-can occur before a RuntimeError is raised.  On PyPy the limit is
-approximative and checked at a lower level.  The default 1000
+can occur before a RecursionError is raised.  On PyPy the limit
+is approximative and checked at a lower level.  The default 1000
 reserves 768KB of stack space, which should suffice (on Linux,
 depending on the compiler settings) for ~1400 calls.  Setting the
 value to N reserves N/1000 times 768KB of stack space.

From pypy.commits at gmail.com  Thu Aug 18 11:07:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 08:07:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Seems that sys.__package__ is also '' on
 CPython 3.5.
Message-ID: <57b5cf31.a6a5c20a.d74e.e602@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86280:123890e04030
Date: 2016-08-18 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/123890e04030/

Log:	Seems that sys.__package__ is also '' on CPython 3.5.

diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -165,7 +165,7 @@
         import sys
         import os
 
-        assert sys.__package__ is None
+        assert sys.__package__ == ''
         assert os.__package__ == ''
         assert not hasattr(type(sys)('foo'), '__package__')
 

From pypy.commits at gmail.com  Thu Aug 18 11:10:19 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 Aug 2016 08:10:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57b5cfdb.6974c20a.af6a3.e61b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r86281:abb8120a073d
Date: 2016-08-18 16:09 +0100
http://bitbucket.org/pypy/pypy/changeset/abb8120a073d/

Log:	hg merge py3k

diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -356,6 +356,11 @@
 that a C-level traceback is usually of no help at all in PyPy.
 Debugging PyPy can be annoying.
 
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
 In more details:
 
 * First, please give the exact PyPy version, and the OS.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -128,3 +128,19 @@
 .. branch: cpyext-realloc
 
 Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1748,6 +1748,23 @@
                 "Python int too large for C unsigned short")
         return value
 
+    def c_uid_t_w(self, w_obj):
+        # xxx assumes that uid_t and gid_t are a C unsigned int.
+        # Equivalent to space.c_uint_w(), with the exception that
+        # it also accepts -1 and converts that to UINT_MAX, which
+        # is (uid_t)-1.  And values smaller than -1 raise
+        # OverflowError, not ValueError.
+        try:
+            return self.c_uint_w(w_obj)
+        except OperationError as e:
+            if e.match(self, self.w_ValueError):
+                # ValueError: cannot convert negative integer to unsigned
+                if self.int_w(w_obj) == -1:
+                    return UINT_MAX
+                raise oefmt(self.w_OverflowError,
+                            "user/group id smaller than minimum (-1)")
+            raise
+
     def truncatedint_w(self, w_obj, allow_conversion=True):
         # Like space.gateway_int_w(), but return the integer truncated
         # instead of raising OverflowError.  For obscure cases only.
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -177,6 +177,9 @@
     def visit_c_ushort(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_c_uid_t(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_truncatedint_w(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -307,6 +310,9 @@
     def visit_c_ushort(self, typ):
         self.run_args.append("space.c_ushort_w(%s)" % (self.scopenext(),))
 
+    def visit_c_uid_t(self, typ):
+        self.run_args.append("space.c_uid_t_w(%s)" % (self.scopenext(),))
+
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
@@ -456,6 +462,9 @@
     def visit_c_ushort(self, typ):
         self.unwrap.append("space.c_ushort_w(%s)" % (self.nextarg(),))
 
+    def visit_c_uid_t(self, typ):
+        self.unwrap.append("space.c_uid_t_w(%s)" % (self.nextarg(),))
+
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -310,11 +310,15 @@
                             self.ctype.name, ct.name)
             #
             itemsize = ct.ctitem.size
-            if itemsize <= 0:
-                itemsize = 1
             with self as ptr1, w_other as ptr2:
                 diff = (rffi.cast(lltype.Signed, ptr1) -
-                        rffi.cast(lltype.Signed, ptr2)) // itemsize
+                        rffi.cast(lltype.Signed, ptr2))
+            if itemsize > 1:
+                if diff % itemsize:
+                    raise oefmt(space.w_ValueError,
+                        "pointer subtraction: the distance between the two "
+                        "pointers is not a multiple of the item size")
+                diff //= itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -257,6 +257,7 @@
     sandboxsafe=True)
 
 # split here for JIT backends that don't support floats/longlongs/etc.
+ at jit.dont_look_inside
 def is_nonnull_longdouble(cdata):
     return _is_nonnull_longdouble(read_raw_longdouble_data(cdata))
 def is_nonnull_float(cdata, size):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -576,6 +576,19 @@
     e = py.test.raises(TypeError, "q - a")
     assert str(e.value) == "cannot subtract cdata 'short *' and cdata 'int *'"
 
+def test_ptr_sub_unaligned():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    a = cast(BIntPtr, 1240)
+    for bi in range(1430, 1438):
+        b = cast(BIntPtr, bi)
+        if ((bi - 1240) % size_of_int()) == 0:
+            assert b - a == (bi - 1240) // size_of_int()
+            assert a - b == (1240 - bi) // size_of_int()
+        else:
+            py.test.raises(ValueError, "b - a")
+            py.test.raises(ValueError, "a - b")
+
 def test_cast_primitive_from_cdata():
     p = new_primitive_type("int")
     n = cast(p, cast(p, -42))
diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,10 +69,21 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        space.appexec([space.wrap(str(tmpdir))], """(path):
+        cls.w_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
             import _cffi_backend     # force it to be initialized
             import sys
             sys.path.insert(0, path)
+            from re_python_pysrc import ffi
+            del sys.path[0]
+            return ffi
+        """)
+        cls.w_sub_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
+            import _cffi_backend     # force it to be initialized
+            import sys
+            sys.path.insert(0, path)
+            from re_py_subsrc import ffi
+            del sys.path[0]
+            return ffi
         """)
 
     def teardown_method(self, meth):
@@ -86,25 +97,25 @@
 
 
     def test_constant_1(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
         assert type(lib.add42) is _cffi_backend.FFI.CData
 
     def test_dlclose(self):
         import _cffi_backend
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
         e = raises(ffi.error, ffi.dlclose, lib)
@@ -115,18 +126,18 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
             assert p.x == 5
@@ -134,13 +145,13 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
-        from re_py_subsrc import ffi
+        ffi = self.sub_ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
         assert ffi.integer_const('k2') == 121212
@@ -153,7 +164,7 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
         p = ffi.addressof(lib, 'globalvar42')
@@ -163,25 +174,25 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == b"hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
-        from re_python_pysrc import ffi
+        ffi = self.ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
         assert str(e.value).startswith(
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -79,7 +79,7 @@
 
 
 class AppTestRecompiler:
-    spaceconfig = dict(usemodules=['_cffi_backend', 'imp'])
+    spaceconfig = dict(usemodules=['_cffi_backend', 'imp', 'cpyext', 'struct'])
 
     def setup_class(cls):
         if cls.runappdirect:
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
@@ -268,22 +268,26 @@
     min = 0;
     max = haystacksize;
 
-    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
+    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
         if (value < haystack[pos].uniseq) {
-            if (max == pos) break;
-            else max = pos;
+            if (max != pos) {
+                max = pos;
+                continue;
+            }
         }
         else if (value > haystack[pos].uniseq) {
-            if (min == pos) break;
-            else min = pos;
+            if (min != pos) {
+                min = pos;
+                continue;
+            }
         }
-        else
-            break;
+        break;
+    }
 
-        if (value == haystack[pos].uniseq)
-            return haystack[pos].code;
-        else
-            return DBCINV;
+    if (value == haystack[pos].uniseq) {
+        return haystack[pos].code;
+    }
+    return DBCINV;
 }
 #endif
 
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -2,6 +2,9 @@
 #define Py_PYTHON_H
 
 /* Compat stuff */
+#ifdef __GNUC__
+#define _GNU_SOURCE 1
+#endif
 #ifndef _WIN32
 # include <inttypes.h>
 # include <stdint.h>
@@ -52,7 +55,6 @@
 #ifndef DL_IMPORT
 #       define DL_IMPORT(RTYPE) RTYPE
 #endif
-
 #include <stdlib.h>
 
 #ifndef _WIN32
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -11,7 +11,7 @@
 from rpython.rlib import rposix, rposix_stat
 from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_longlong, intmask
+from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -28,20 +28,21 @@
 
 c_int = "c_int"
 
-# CPython 2.7 semantics are too messy to follow exactly,
-# e.g. setuid(-2) works on 32-bit but not on 64-bit.  As a result,
-# we decided to just accept any 'int', i.e. any C signed long, and
-# check that they are in range(-2**31, 2**32).  In other words, we
-# accept any number that is either a signed or an unsigned C int.
-c_uid_t = int
-c_gid_t = int
-if sys.maxint == 2147483647:
-    def check_uid_range(space, num):
-        pass
-else:
-    def check_uid_range(space, num):
-        if num < -(1 << 31) or num >= (1 << 32):
-            raise oefmt(space.w_OverflowError, "integer out of range")
+# CPython 2.7 semantics used to be too messy, differing on 32-bit vs
+# 64-bit, but this was cleaned up in recent 2.7.x.  Now, any function
+# taking a uid_t or gid_t accepts numbers in range(-1, 2**32) as an
+# r_uint, with -1 being equivalent to 2**32-1.  Any function that
+# returns a uid_t or gid_t returns either an int or a long, depending
+# on whether it fits or not, but always positive.
+c_uid_t = 'c_uid_t'
+c_gid_t = 'c_uid_t'
+
+def wrap_uid(space, uid):
+    if uid <= r_uint(sys.maxint):
+        return space.wrap(intmask(uid))
+    else:
+        return space.wrap(uid)     # an unsigned number
+wrap_gid = wrap_uid
 
 class FileEncoder(object):
     is_unicode = True
@@ -1530,7 +1531,7 @@
 
     Return the current process's user id.
     """
-    return space.wrap(os.getuid())
+    return wrap_uid(space, os.getuid())
 
 @unwrap_spec(arg=c_uid_t)
 def setuid(space, arg):
@@ -1538,12 +1539,10 @@
 
     Set the current process's user id.
     """
-    check_uid_range(space, arg)
     try:
         os.setuid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_uid_t)
 def seteuid(space, arg):
@@ -1551,12 +1550,10 @@
 
     Set the current process's effective user id.
     """
-    check_uid_range(space, arg)
     try:
         os.seteuid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_gid_t)
 def setgid(space, arg):
@@ -1564,12 +1561,10 @@
 
     Set the current process's group id.
     """
-    check_uid_range(space, arg)
     try:
         os.setgid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_gid_t)
 def setegid(space, arg):
@@ -1577,12 +1572,10 @@
 
     Set the current process's effective group id.
     """
-    check_uid_range(space, arg)
     try:
         os.setegid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(path='fsencode')
 def chroot(space, path):
@@ -1601,21 +1594,21 @@
 
     Return the current process's group id.
     """
-    return space.wrap(os.getgid())
+    return wrap_gid(space, os.getgid())
 
 def getegid(space):
     """ getegid() -> gid
 
     Return the current process's effective group id.
     """
-    return space.wrap(os.getegid())
+    return wrap_gid(space, os.getegid())
 
 def geteuid(space):
     """ geteuid() -> euid
 
     Return the current process's effective user id.
     """
-    return space.wrap(os.geteuid())
+    return wrap_uid(space, os.geteuid())
 
 def getgroups(space):
     """ getgroups() -> list of group IDs
@@ -1626,7 +1619,7 @@
         list = os.getgroups()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newlist([space.wrap(e) for e in list])
+    return space.newlist([wrap_gid(space, e) for e in list])
 
 def setgroups(space, w_list):
     """ setgroups(list)
@@ -1635,9 +1628,7 @@
     """
     list = []
     for w_gid in space.unpackiterable(w_list):
-        gid = space.int_w(w_gid)
-        check_uid_range(space, gid)
-        list.append(gid)
+        list.append(space.c_uid_t_w(w_gid))
     try:
         os.setgroups(list[:])
     except OSError as e:
@@ -1711,13 +1702,10 @@
 
     Set the current process's real and effective user ids.
     """
-    check_uid_range(space, ruid)
-    check_uid_range(space, euid)
     try:
         os.setreuid(ruid, euid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(rgid=c_gid_t, egid=c_gid_t)
 def setregid(space, rgid, egid):
@@ -1725,13 +1713,10 @@
 
     Set the current process's real and effective group ids.
     """
-    check_uid_range(space, rgid)
-    check_uid_range(space, egid)
     try:
         os.setregid(rgid, egid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(pid=c_int)
 def getsid(space, pid):
@@ -1788,9 +1773,9 @@
         (ruid, euid, suid) = os.getresuid()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(ruid),
-                           space.wrap(euid),
-                           space.wrap(suid)])
+    return space.newtuple([wrap_uid(space, ruid),
+                           wrap_uid(space, euid),
+                           wrap_uid(space, suid)])
 
 def getresgid(space):
     """ getresgid() -> (rgid, egid, sgid)
@@ -1801,9 +1786,9 @@
         (rgid, egid, sgid) = os.getresgid()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(rgid),
-                           space.wrap(egid),
-                           space.wrap(sgid)])
+    return space.newtuple([wrap_gid(space, rgid),
+                           wrap_gid(space, egid),
+                           wrap_gid(space, sgid)])
 
 @unwrap_spec(ruid=c_uid_t, euid=c_uid_t, suid=c_uid_t)
 def setresuid(space, ruid, euid, suid):
@@ -1925,8 +1910,6 @@
   an open file descriptor.
 dir_fd and follow_symlinks may not be implemented on your platform.
   If they are unavailable, using them will raise a NotImplementedError."""
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     if not (rposix.HAVE_LCHOWN or rposix.HAVE_FCHMODAT):
         if not follow_symlinks:
             raise argument_unavailable(space, 'chown', 'follow_symlinks')
@@ -1972,8 +1955,6 @@
 Change the owner and group id of path to the numeric uid and gid.
 This function will not follow symbolic links.
 Equivalent to os.chown(path, uid, gid, follow_symlinks=False)."""
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.lchown(path, uid, gid)
     except OSError as e:
@@ -1986,8 +1967,6 @@
 Change the owner and group id of the file given by file descriptor
 fd to the numeric uid and gid.  Equivalent to os.chown(fd, uid, gid)."""
     fd = space.c_filedescriptor_w(w_fd)
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.fchown(fd, uid, gid)
     except OSError as e:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -595,8 +595,9 @@
     if hasattr(os, 'setuid'):
         def test_os_setuid_error(self):
             os = self.posix
-            raises(OverflowError, os.setuid, -2**31-1)
+            raises(OverflowError, os.setuid, -2)
             raises(OverflowError, os.setuid, 2**32)
+            raises(OSError, os.setuid, -1)
 
     if hasattr(os, 'getgid'):
         def test_os_getgid(self):
@@ -641,8 +642,10 @@
     if hasattr(os, 'setgid'):
         def test_os_setgid_error(self):
             os = self.posix
-            raises(OverflowError, os.setgid, -2**31-1)
+            raises(OverflowError, os.setgid, -2)
             raises(OverflowError, os.setgid, 2**32)
+            raises(OSError, os.setgid, -1)
+            raises(OSError, os.setgid, 2**32-1)
 
     if hasattr(os, 'getsid'):
         def test_os_getsid(self):
diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -45,5 +45,8 @@
 
 def test_setrlimit():
     # minimal "does not crash" test
-    x = resource.getrlimit(resource.RLIMIT_CPU)
-    resource.setrlimit(resource.RLIMIT_CPU, x)
+    x, y = resource.getrlimit(resource.RLIMIT_CPU)
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))
+    x += 0.2
+    y += 0.3
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))    # truncated to ints
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -221,7 +221,6 @@
         stack_index = 0
         while True:
             current = self
-            number_to_readd = 0
             number_to_readd, attr = self._find_branch_to_move_into(name, index)
             # we found the attributes further up, need to save the
             # previous values of the attributes we passed
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
+cffi>=1.4.0
+
 # hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2
diff --git a/rpython/doc/arm.rst b/rpython/doc/arm.rst
--- a/rpython/doc/arm.rst
+++ b/rpython/doc/arm.rst
@@ -148,7 +148,7 @@
 
 ::
 
-  pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O1 --platform=arm target.py
+  pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O2 --platform=arm target.py
 
 If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``.
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1051,8 +1051,9 @@
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     #
+    jd_name = jitdriver_sd.jitdriver.name
     metainterp_sd.jitlog.start_new_trace(metainterp_sd,
-            faildescr=resumekey, entry_bridge=False)
+            faildescr=resumekey, entry_bridge=False, jd_name=jd_name)
     #
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -98,6 +98,7 @@
     def log_abort_loop(self, trace, memo=None):
         debug_start("jit-abort-log")
         if not have_debug_prints():
+            debug_stop("jit-abort-log")
             return
         inputargs, operations = self._unpack_trace(trace)
         logops = self._log_operations(inputargs, operations, ops_offset=None,
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -64,6 +64,10 @@
     testself.all_graphs = graphs
     result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0]
 
+
+    class FakeJitDriver:
+        name = 'fakejitdriver'
+
     class FakeJitDriverSD:
         num_green_args = 0
         portal_graph = graphs[0]
@@ -72,6 +76,7 @@
         result_type = result_kind
         portal_runner_ptr = "???"
         vec = False
+        jitdriver = FakeJitDriver()
 
     stats = history.Stats(None)
     cpu = CPUClass(rtyper, stats, None, False)
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 1
+JITLOG_VERSION = 2
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -331,7 +331,7 @@
     def finish(self):
         jitlog_teardown()
 
-    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False):
+    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False, jd_name=""):
         # even if the logger is not enabled, increment the trace id
         self.trace_id += 1
         if not jitlog_enabled():
@@ -345,6 +345,7 @@
         else:
             content.append(encode_str('loop'))
             content.append(encode_le_addr(int(entry_bridge)))
+        content.append(encode_str(jd_name))
         self._write_marked(MARK_START_TRACE, ''.join(content))
 
     def trace_aborted(self):
@@ -508,7 +509,9 @@
         """ an operation is written as follows:
             <marker> <opid (16 bit)> \
                      <len (32 bit)> \
-                     <res_val>,<arg_0>,...,<arg_n>,<descr>
+                     <res_val>,<arg_0>,...,<arg_n> \
+                     <descr>
+                     <failarg_0>,...<failarg_n>
             The marker indicates if the last argument is
             a descr or a normal argument.
         """
@@ -517,16 +520,21 @@
         le_opnum = encode_le_16bit(op.getopnum())
         str_res = self.var_to_str(op)
         line = ','.join([str_res] + str_args)
+        failargslist = op.getfailargs()
+        failargs = ''
+        if failargslist:
+            failargs = ','.join([self.var_to_str(farg) for farg in failargslist])
+        #
         if descr:
             descr_str = descr.repr_of_descr()
             line = line + ',' + descr_str
             string = encode_str(line)
             descr_number = compute_unique_id(descr)
             le_descr_number = encode_le_addr(descr_number)
-            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number
+            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number + encode_str(failargs)
         else:
             string = encode_str(line)
-            return MARK_RESOP, le_opnum + string
+            return MARK_RESOP, le_opnum + string + encode_str(failargs)
 
 
     def write_core_dump(self, operations, i, op, ops_offset):
@@ -578,6 +586,8 @@
         return ''.join(dump)
 
     def var_to_str(self, arg):
+        if arg is None:
+            return '-'
         try:
             mv = self.memo[arg]
         except KeyError:
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -48,7 +48,7 @@
         file.ensure()
         fd = file.open('wb')
         jl.jitlog_init(fd.fileno())
-        logger.start_new_trace(self.make_metainterp_sd())
+        logger.start_new_trace(self.make_metainterp_sd(), jd_name='jdname')
         log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
         op = ResOperation(rop.DEBUG_MERGE_POINT, [ConstInt(0), ConstInt(0), ConstInt(0)])
         log_trace.write([], [op])
@@ -58,6 +58,7 @@
         is_32bit = chr(sys.maxint == 2**31-1)
         assert binary == (jl.MARK_START_TRACE) + jl.encode_le_addr(1) + \
                          jl.encode_str('loop') + jl.encode_le_addr(0) + \
+                         jl.encode_str('jdname') + \
                          (jl.MARK_TRACE) + jl.encode_le_addr(1) + \
                          (jl.MARK_INPUT_ARGS) + jl.encode_str('') + \
                          (jl.MARK_INIT_MERGE_POINT) + b'\x05\x00\x01s\x00i\x08s\x00i\x10s' + \
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -96,12 +96,15 @@
             return 0;
         }
     ''',]
+    post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);']
 else:
     separate_module_sources = []
+    post_include_bits = []
     includes=['errno.h','stdio.h']
 errno_eci = ExternalCompilationInfo(
     includes=includes,
     separate_module_sources=separate_module_sources,
+    post_include_bits=post_include_bits,
 )
 
 # Direct getters/setters, don't use directly!
@@ -251,6 +254,8 @@
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
     if not _WIN32:
+        UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
+        GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
         CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
 
         TMS = rffi_platform.Struct(
@@ -1450,32 +1455,33 @@
 def setpgid(pid, gid):
     handle_posix_error('setpgid', c_setpgid(pid, gid))
 
-PID_GROUPS_T = rffi.CArrayPtr(rffi.PID_T)
-c_getgroups = external('getgroups', [rffi.INT, PID_GROUPS_T], rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setgroups = external('setgroups', [rffi.SIZE_T, PID_GROUPS_T], rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_initgroups = external('initgroups', [rffi.CCHARP, rffi.PID_T], rffi.INT,
-                        save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    GID_GROUPS_T = rffi.CArrayPtr(GID_T)
+    c_getgroups = external('getgroups', [rffi.INT, GID_GROUPS_T], rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setgroups = external('setgroups', [rffi.SIZE_T, GID_GROUPS_T], rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_initgroups = external('initgroups', [rffi.CCHARP, GID_T], rffi.INT,
+                            save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('getgroups')
 def getgroups():
     n = handle_posix_error('getgroups',
-                           c_getgroups(0, lltype.nullptr(PID_GROUPS_T.TO)))
-    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+                           c_getgroups(0, lltype.nullptr(GID_GROUPS_T.TO)))
+    groups = lltype.malloc(GID_GROUPS_T.TO, n, flavor='raw')
     try:
         n = handle_posix_error('getgroups', c_getgroups(n, groups))
-        return [widen(groups[i]) for i in range(n)]
+        return [widen_gid(groups[i]) for i in range(n)]
     finally:
         lltype.free(groups, flavor='raw')
 
 @replace_os_function('setgroups')
 def setgroups(gids):
     n = len(gids)
-    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+    groups = lltype.malloc(GID_GROUPS_T.TO, n, flavor='raw')
     try:
         for i in range(n):
-            groups[i] = rffi.cast(rffi.PID_T, gids[i])
+            groups[i] = rffi.cast(GID_T, gids[i])
         handle_posix_error('setgroups', c_setgroups(n, groups))
     finally:
         lltype.free(groups, flavor='raw')
@@ -1526,104 +1532,115 @@
 
 #___________________________________________________________________
 
-c_getuid = external('getuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_geteuid = external('geteuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_setuid = external('setuid', [rffi.INT], rffi.INT,
-                    save_err=rffi.RFFI_SAVE_ERRNO)
-c_seteuid = external('seteuid', [rffi.INT], rffi.INT,
-                     save_err=rffi.RFFI_SAVE_ERRNO)
-c_getgid = external('getgid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_getegid = external('getegid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_setgid = external('setgid', [rffi.INT], rffi.INT,
-                    save_err=rffi.RFFI_SAVE_ERRNO)
-c_setegid = external('setegid', [rffi.INT], rffi.INT,
-                     save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    c_getuid = external('getuid', [], UID_T)
+    c_geteuid = external('geteuid', [], UID_T)
+    c_setuid = external('setuid', [UID_T], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+    c_seteuid = external('seteuid', [UID_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    c_getgid = external('getgid', [], GID_T)
+    c_getegid = external('getegid', [], GID_T)
+    c_setgid = external('setgid', [GID_T], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setegid = external('setegid', [GID_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 
- at replace_os_function('getuid')
-def getuid():
-    return handle_posix_error('getuid', c_getuid())
+    def widen_uid(x):
+        return rffi.cast(lltype.Unsigned, x)
+    widen_gid = widen_uid
 
- at replace_os_function('geteuid')
-def geteuid():
-    return handle_posix_error('geteuid', c_geteuid())
+    # NOTE: the resulting type of functions that return a uid/gid is
+    # always Unsigned.  The argument type of functions that take a
+    # uid/gid should also be Unsigned.
 
- at replace_os_function('setuid')
-def setuid(uid):
-    handle_posix_error('setuid', c_setuid(uid))
+    @replace_os_function('getuid')
+    def getuid():
+        return widen_uid(c_getuid())
 
- at replace_os_function('seteuid')
-def seteuid(uid):
-    handle_posix_error('seteuid', c_seteuid(uid))
+    @replace_os_function('geteuid')
+    def geteuid():
+        return widen_uid(c_geteuid())
 
- at replace_os_function('getgid')
-def getgid():
-    return handle_posix_error('getgid', c_getgid())
+    @replace_os_function('setuid')
+    def setuid(uid):
+        handle_posix_error('setuid', c_setuid(uid))
 
- at replace_os_function('getegid')
-def getegid():
-    return handle_posix_error('getegid', c_getegid())
+    @replace_os_function('seteuid')
+    def seteuid(uid):
+        handle_posix_error('seteuid', c_seteuid(uid))
 
- at replace_os_function('setgid')
-def setgid(gid):
-    handle_posix_error('setgid', c_setgid(gid))
+    @replace_os_function('getgid')
+    def getgid():
+        return widen_gid(c_getgid())
 
- at replace_os_function('setegid')
-def setegid(gid):
-    handle_posix_error('setegid', c_setegid(gid))
+    @replace_os_function('getegid')
+    def getegid():
+        return widen_gid(c_getegid())
 
-c_setreuid = external('setreuid', [rffi.INT, rffi.INT], rffi.INT,
-                      save_err=rffi.RFFI_SAVE_ERRNO)
-c_setregid = external('setregid', [rffi.INT, rffi.INT], rffi.INT,
-                      save_err=rffi.RFFI_SAVE_ERRNO)
+    @replace_os_function('setgid')
+    def setgid(gid):
+        handle_posix_error('setgid', c_setgid(gid))
 
- at replace_os_function('setreuid')
-def setreuid(ruid, euid):
-    handle_posix_error('setreuid', c_setreuid(ruid, euid))
+    @replace_os_function('setegid')
+    def setegid(gid):
+        handle_posix_error('setegid', c_setegid(gid))
 
- at replace_os_function('setregid')
-def setregid(rgid, egid):
-    handle_posix_error('setregid', c_setregid(rgid, egid))
+    c_setreuid = external('setreuid', [UID_T, UID_T], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setregid = external('setregid', [GID_T, GID_T], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
-c_getresuid = external('getresuid', [rffi.INTP] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_getresgid = external('getresgid', [rffi.INTP] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setresuid = external('setresuid', [rffi.INT] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setresgid = external('setresgid', [rffi.INT] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
+    @replace_os_function('setreuid')
+    def setreuid(ruid, euid):
+        handle_posix_error('setreuid', c_setreuid(ruid, euid))
 
- at replace_os_function('getresuid')
-def getresuid():
-    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-    try:
-        handle_posix_error('getresuid',
-                           c_getresuid(rffi.ptradd(out, 0),
-                                       rffi.ptradd(out, 1),
-                                       rffi.ptradd(out, 2)))
-        return (widen(out[0]), widen(out[1]), widen(out[2]))
-    finally:
-        lltype.free(out, flavor='raw')
+    @replace_os_function('setregid')
+    def setregid(rgid, egid):
+        handle_posix_error('setregid', c_setregid(rgid, egid))
 
- at replace_os_function('getresgid')
-def getresgid():
-    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-    try:
-        handle_posix_error('getresgid',
-                           c_getresgid(rffi.ptradd(out, 0),
-                                       rffi.ptradd(out, 1),
-                                       rffi.ptradd(out, 2)))
-        return (widen(out[0]), widen(out[1]), widen(out[2]))
-    finally:
-        lltype.free(out, flavor='raw')
+    UID_T_P = lltype.Ptr(lltype.Array(UID_T, hints={'nolength': True}))
+    GID_T_P = lltype.Ptr(lltype.Array(GID_T, hints={'nolength': True}))
+    c_getresuid = external('getresuid', [UID_T_P] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_getresgid = external('getresgid', [GID_T_P] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setresuid = external('setresuid', [UID_T] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setresgid = external('setresgid', [GID_T] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
 
- at replace_os_function('setresuid')
-def setresuid(ruid, euid, suid):
-    handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+    @replace_os_function('getresuid')
+    def getresuid():
+        out = lltype.malloc(UID_T_P.TO, 3, flavor='raw')
+        try:
+            handle_posix_error('getresuid',
+                               c_getresuid(rffi.ptradd(out, 0),
+                                           rffi.ptradd(out, 1),
+                                           rffi.ptradd(out, 2)))
+            return (widen_uid(out[0]), widen_uid(out[1]), widen_uid(out[2]))
+        finally:
+            lltype.free(out, flavor='raw')
 
- at replace_os_function('setresgid')
-def setresgid(rgid, egid, sgid):
-    handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
+    @replace_os_function('getresgid')
+    def getresgid():
+        out = lltype.malloc(GID_T_P.TO, 3, flavor='raw')
+        try:
+            handle_posix_error('getresgid',
+                               c_getresgid(rffi.ptradd(out, 0),
+                                           rffi.ptradd(out, 1),
+                                           rffi.ptradd(out, 2)))
+            return (widen_gid(out[0]), widen_gid(out[1]), widen_gid(out[2]))
+        finally:
+            lltype.free(out, flavor='raw')
+
+    @replace_os_function('setresuid')
+    def setresuid(ruid, euid, suid):
+        handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+
+    @replace_os_function('setresgid')
+    def setresgid(rgid, egid, sgid):
+        handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
 
 #___________________________________________________________________
 
diff --git a/rpython/rlib/rweakref.py b/rpython/rlib/rweakref.py
--- a/rpython/rlib/rweakref.py
+++ b/rpython/rlib/rweakref.py
@@ -142,7 +142,7 @@
 
     def compute_result_annotation(self, s_keyclass, s_valueclass):
         assert s_keyclass.is_constant()
-        s_key = self.bookkeeper.immutablevalue(s_keyclass.const())
+        s_key = self.bookkeeper.valueoftype(s_keyclass.const)
         return SomeWeakValueDict(
             s_key,
             _getclassdef(s_valueclass))
@@ -158,7 +158,7 @@
         bk = self.bookkeeper
         x = self.instance
         return SomeWeakValueDict(
-            bk.immutablevalue(x._keyclass()),
+            bk.valueoftype(x._keyclass),
             bk.getuniqueclassdef(x._valueclass))
 
 def _getclassdef(s_instance):
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -180,3 +180,36 @@
         RWeakValueDictionary(str, X).get("foobar")
         RWeakValueDictionary(int, Y).get(42)
     interpret(g, [])
+
+def test_key_instance():
+    class K(object):
+        pass
+    keys = [K(), K(), K()]
+
+    def g(d):
+        assert d.get(keys[3]) is None
+        x1 = X(); x2 = X(); x3 = X()
+        d.set(keys[0], x1)
+        d.set(keys[1], x2)
+        d.set(keys[2], x3)
+        assert d.get(keys[0]) is x1
+        assert d.get(keys[1]) is x2
+        assert d.get(keys[2]) is x3
+        assert d.get(keys[3]) is None
+        return x1, x3    # x2 dies
+    def f():
+        keys.append(K())
+        d = RWeakValueDictionary(K, X)
+        x1, x3 = g(d)
+        rgc.collect(); rgc.collect()
+        assert d.get(keys[0]) is x1
+        assert d.get(keys[1]) is None
+        assert d.get(keys[2]) is x3
+        assert d.get(keys[3]) is None
+        d.set(keys[0], None)
+        assert d.get(keys[0]) is None
+        assert d.get(keys[1]) is None
+        assert d.get(keys[2]) is x3
+        assert d.get(keys[3]) is None
+    f()
+    interpret(f, [])

From pypy.commits at gmail.com  Thu Aug 18 11:34:30 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 18 Aug 2016 08:34:30 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: test_buffer_protocol
 passes now,
 allocating a new Py_buffer that should be managed by the CPyBuffer object
 (needs some discussion)
Message-ID: <57b5d586.482cc20a.97c42.f15e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86282:c6decd039ce0
Date: 2016-08-18 17:33 +0200
http://bitbucket.org/pypy/pypy/changeset/c6decd039ce0/

Log:	test_buffer_protocol passes now, allocating a new Py_buffer that
	should be managed by the CPyBuffer object (needs some discussion)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -649,6 +649,7 @@
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
         ))
+Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -5,13 +5,13 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl)
+    mangle_name, pypy_decl, Py_buffer)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    readbufferproc, ssizessizeobjargproc)
+    readbufferproc, getbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -302,6 +302,7 @@
     _immutable_ = True
 
     def __init__(self, ptr, size, w_obj):
+        # XXX leak of ptr
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
@@ -317,8 +318,7 @@
         return rffi.cast(rffi.CCHARP, self.ptr)
 
     def getformat(self):
-        import pdb; pdb.set_trace()
-        return 'i'
+        return rffi.charp2str(self.ptr.c_format)
 
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
@@ -330,8 +330,15 @@
         return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
 
 def wrap_getbuffer(space, w_self, w_args, func):
-    import pdb; pdb.set_trace()
-    return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
+    func_target = rffi.cast(getbufferproc, func)
+    # XXX leak
+    pybuf = lltype.malloc(Py_buffer, flavor='raw', track_allocation=False)
+    # XXX flags are not in w_args?
+    flags = rffi.cast(rffi.INT_real,0)
+    size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+    if size < 0:
+        space.fromcache(State).check_and_raise_exception(always=True)
+    return space.newbuffer(CPyBuffer(pybuf, size, w_self))
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -5,6 +5,7 @@
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
+from pypy.module.cpyext.api import Py_bufferP
 
 
 P, FT, PyO = Ptr, FuncType, PyObject
@@ -58,8 +59,7 @@
 writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
 segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
 charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
-## We don't support new buffer interface for now
-getbufferproc = rffi.VOIDP
+getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = rffi.VOIDP
 
 

From pypy.commits at gmail.com  Thu Aug 18 11:36:47 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 Aug 2016 08:36:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix constant-folding for '**'
Message-ID: <57b5d60f.08d11c0a.c6951.0aaf@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86283:9e01cc8cdef4
Date: 2016-08-18 16:36 +0100
http://bitbucket.org/pypy/pypy/changeset/9e01cc8cdef4/

Log:	Fix constant-folding for '**'

diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -118,7 +118,7 @@
     # don't constant-fold if "w_left" and "w_right" are integers and
     # the estimated bit length of the power is unreasonably large
     space.appexec([w_left, w_right], """(left, right):
-        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+        if isinstance(left, int) and isinstance(right, int):
             if left.bit_length() * right > 5000:
                 raise OverflowError
     """)

From pypy.commits at gmail.com  Thu Aug 18 12:38:39 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 Aug 2016 09:38:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: Use explicit types,
 not space.wrap() (fixes the test on py3k)
Message-ID: <57b5e48f.8aacc20a.9e40d.0622@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86284:77bc9e962a4e
Date: 2016-08-18 17:38 +0100
http://bitbucket.org/pypy/pypy/changeset/77bc9e962a4e/

Log:	Use explicit types, not space.wrap() (fixes the test on py3k)

diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -10,10 +10,10 @@
 
     def setup_class(cls):
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__jitlog.1')))
-        cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
-        cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
+        cls.w_mark_header = cls.space.newbytes(jl.MARK_JITLOG_HEADER)
+        cls.w_version = cls.space.newbytes(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
-        cls.w_machine = cls.space.wrap(platform.machine())
+        cls.w_machine = cls.space.newbytes(platform.machine())
         cls.w_resops = cls.space.newdict()
         space = cls.space
         for key, value in opname.items():
@@ -48,5 +48,3 @@
                 assert opnum in self.resops
                 # the name must equal
                 assert self.resops[opnum] == opname
-
-

From pypy.commits at gmail.com  Thu Aug 18 13:16:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 10:16:15 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add a more precise assert: the computed
 stack depth must never be
Message-ID: <57b5ed5f.0cce1c0a.dc8d9.2c57@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86285:b4e83acb5f3c
Date: 2016-08-18 19:15 +0200
http://bitbucket.org/pypy/pypy/changeset/b4e83acb5f3c/

Log:	Add a more precise assert: the computed stack depth must never be
	negative. Fix the logic to avoid computing stack depths for
	unreachable blocks.

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -387,7 +387,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -406,8 +407,11 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            assert depth >= 0
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode

From pypy.commits at gmail.com  Thu Aug 18 13:34:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 10:34:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Manual copy of b4e83acb5f3c,
 and fix of a resulting issue: two opcodes
Message-ID: <57b5f1b6.497bc20a.7638e.140b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86286:f8ffd7d38ab1
Date: 2016-08-18 19:34 +0200
http://bitbucket.org/pypy/pypy/changeset/f8ffd7d38ab1/

Log:	Manual copy of b4e83acb5f3c, and fix of a resulting issue: two
	opcodes had a wrong stack effect

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -389,7 +389,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -408,8 +409,11 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            assert depth >= 0
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode
@@ -560,7 +564,6 @@
     ops.LIST_APPEND: -1,
     ops.SET_ADD: -1,
     ops.MAP_ADD: -2,
-    # XXX 
 
     ops.BINARY_POWER: -1,
     ops.BINARY_MULTIPLY: -1,
@@ -602,8 +605,8 @@
 
     ops.PRINT_EXPR: -1,
 
-    ops.WITH_CLEANUP_START: -1,
-    ops.WITH_CLEANUP_FINISH: -1,  # XXX Sometimes more
+    ops.WITH_CLEANUP_START: 1,
+    ops.WITH_CLEANUP_FINISH: -2,
     ops.LOAD_BUILD_CLASS: 1,
     ops.POP_BLOCK: 0,
     ops.POP_EXCEPT: -1,
@@ -619,7 +622,6 @@
     ops.YIELD_FROM: -1,
     ops.COMPARE_OP: -1,
 
-    # TODO 
     ops.LOOKUP_METHOD: 1,
 
     ops.LOAD_NAME: 1,
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -511,6 +511,9 @@
         x *= 7
         """, 'x', 42
 
+    def test_with_stacksize_bug(self):
+        compile_with_astcompiler("with a:\n  pass", 'exec', self.space)
+
     def test_with_bug(self):
         yield self.simple_test, """
         class ContextManager:

From pypy.commits at gmail.com  Thu Aug 18 13:58:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 10:58:56 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: RecursionError done
Message-ID: <57b5f760.45c8c20a.762ba.208e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5676:9f9502d1981e
Date: 2016-08-18 19:58 +0200
http://bitbucket.org/pypy/extradoc/changeset/9f9502d1981e/

Log:	RecursionError done

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -51,7 +51,7 @@
 * Generators have a new gi_yieldfrom attribute
 
 * A new RecursionError exception is now raised when maximum recursion
-  depth is reached.
+  depth is reached. (DONE)
 
 * The new os.scandir() function
 

From pypy.commits at gmail.com  Thu Aug 18 14:17:39 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 18 Aug 2016 11:17:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-async: Check for correct values in 'async
 for' test
Message-ID: <57b5fbc3.109a1c0a.aeffb.43c3@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5-async
Changeset: r86287:5f146fc69289
Date: 2016-08-18 20:17 +0200
http://bitbucket.org/pypy/pypy/changeset/5f146fc69289/

Log:	Check for correct values in 'async for' test

diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -19,17 +19,14 @@
 
 loop = asyncio.get_event_loop()
 loop.run_until_complete(f())
-print("done with async loop")
         """
     
     def test_async_for(self):
-        # temporary test from
-        # http://blog.idego.pl/2015/12/05/back-to-the-future-with-async-and-await-in-python-3-5/
+        # tests if async for receives all stores values in the right order
+        # and if the correct methods __aiter__ and __anext__ get called
+        # and if the end results of run_until_complete are None (in a tuple)
         """
 import asyncio
-import logging
-import sys
-logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(asctime)s: %(message)s")
 
 class AsyncIter:
     def __init__(self):
@@ -46,13 +43,32 @@
             return self._data[self._index-1]
         raise StopAsyncIteration
 
-async def do_loop():
-    async for x in AsyncIter():
-        logging.info(x)
+class Corotest(object):
+    def __init__(self):
+        self.res = "-"
+    
+    async def do_loop(self):
+        async for x in AsyncIter():
+            self.res += str(x)
+            self.res += "-"
 
+cor = Corotest()
 loop = asyncio.get_event_loop()
-futures = [asyncio.ensure_future(do_loop()), asyncio.ensure_future(do_loop())]
-loop.run_until_complete(asyncio.wait(futures))
+futures = [asyncio.ensure_future(cor.do_loop()), asyncio.ensure_future(cor.do_loop())]
+taskres = loop.run_until_complete(asyncio.wait(futures))
+assert cor.res.count('0') == 2
+assert cor.res.count('1') == 2
+assert cor.res.count('2') == 2
+assert cor.res.count('3') == 2
+assert cor.res.count('4') == 2
+assert cor.res.find("0") < cor.res.find("1")
+assert cor.res.find("1") < cor.res.find("2")
+assert cor.res.find("2") < cor.res.find("3")
+assert cor.res.find("3") < cor.res.find("4")
+assert isinstance(taskres, tuple)
+assert len(taskres) == 2
+assert "result=None" in repr(taskres[0].pop())
+assert "result=None" in repr(taskres[0].pop())
         """
     
     def test_asynchronous_context_managers(self):

From pypy.commits at gmail.com  Thu Aug 18 14:23:16 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 18 Aug 2016 11:23:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Merge with py3.5-async
Message-ID: <57b5fd14.88711c0a.a517c.46cb@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86288:10f2791aaf2d
Date: 2016-08-18 20:22 +0200
http://bitbucket.org/pypy/pypy/changeset/10f2791aaf2d/

Log:	Merge with py3.5-async

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -609,7 +609,7 @@
     ops.WITH_CLEANUP_FINISH: -2,
     ops.LOAD_BUILD_CLASS: 1,
     ops.POP_BLOCK: 0,
-    ops.POP_EXCEPT: -1,
+    ops.POP_EXCEPT: -2,
     ops.END_FINALLY: -4,     # assume always 4: we pretend that SETUP_FINALLY
                              # pushes 4.  In truth, it would only push 1 and
                              # the corresponding END_FINALLY only pops 1.
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -599,17 +599,21 @@
         b_try_cleanup = self.new_block()
         b_after_loop = self.new_block()
         b_after_loop_else = self.new_block()
+        
         self.emit_jump(ops.SETUP_LOOP, b_after_loop)
         self.push_frame_block(F_BLOCK_LOOP, b_try)
+        
         fr.iter.walkabout(self)
         self.emit_op(ops.GET_AITER)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
+        
         self.use_next_block(b_try)
         # This adds another line, so each for iteration can be traced.
         self.lineno_set = False
         self.emit_jump(ops.SETUP_EXCEPT, b_except)
         self.push_frame_block(F_BLOCK_EXCEPT, b_try)
+        
         self.emit_op(ops.GET_ANEXT)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
@@ -617,9 +621,10 @@
         self.emit_op(ops.POP_BLOCK)
         self.pop_frame_block(F_BLOCK_EXCEPT, b_try)
         self.emit_jump(ops.JUMP_FORWARD, b_after_try)
+        
         self.use_next_block(b_except)
-        self.emit_op(ops.POP_TOP)
-        self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopIterError")
+        self.emit_op(ops.DUP_TOP)
+        self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopAsyncIteration")
         self.emit_op_arg(ops.COMPARE_OP, 10)
         self.emit_jump(ops.POP_JUMP_IF_FALSE, b_try_cleanup, True)
         
@@ -632,13 +637,17 @@
         
         self.use_next_block(b_try_cleanup)
         self.emit_op(ops.END_FINALLY)
+        
         self.use_next_block(b_after_try)
         self.visit_sequence(fr.body)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_try, True)
+        
         self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP
         self.pop_frame_block(F_BLOCK_LOOP, b_try)
+        
         self.use_next_block(b_after_loop)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_end, True)
+        
         self.use_next_block(b_after_loop_else)
         self.visit_sequence(fr.orelse)
         
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -798,10 +798,15 @@
 )
 assert not GeneratorIterator.typedef.acceptable_as_base_class  # no __new__
 
+# TODO: to have the same distinction (Coroutine | Iterator) as in cpython 3.5,
+# a wrapper typedef with __anext__ has to be created, and __anext__ has to be
+# removed in coroutine
 Coroutine.typedef = TypeDef("coroutine",
     __repr__   = interp2app(Coroutine.descr__repr__),
     __reduce__   = interp2app(Coroutine.descr__reduce__),
     __setstate__ = interp2app(Coroutine.descr__setstate__),
+    __anext__   = interp2app(Coroutine.descr_next,
+                            descrmismatch='__anext__'),
     send       = interp2app(Coroutine.descr_send,
                             descrmismatch='send'),
     throw      = interp2app(Coroutine.descr_throw,
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -1,4 +1,5 @@
 class AppTestAsyncIO(object):
+    """These tests are based on the async-await syntax of Python 3.5."""
     
     spaceconfig = dict(usemodules=["select","_socket","thread","signal",
                                    "struct","_multiprocessing","array",
@@ -9,17 +10,71 @@
         # the problem occured at await asyncio.open_connection
         # after calling run_until_complete
         """
-        import encodings.idna
-        import asyncio
-        async def f():
-            reader, writer = await asyncio.open_connection('example.com', 80)
-        
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(f())
-        print("done with async loop")
+import encodings.idna
+import asyncio
+
+async def f():
+    reader, writer = await asyncio.open_connection('example.com', 80)
+    writer.close()
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(f())
+        """
+    
+    def test_async_for(self):
+        # tests if async for receives all stores values in the right order
+        # and if the correct methods __aiter__ and __anext__ get called
+        # and if the end results of run_until_complete are None (in a tuple)
+        """
+import asyncio
+
+class AsyncIter:
+    def __init__(self):
+        self._data = list(range(5))
+        self._index = 0
+    
+    async def __aiter__(self):
+        return self
+    
+    async def __anext__(self):
+        while self._index < 5:
+            await asyncio.sleep(1)
+            self._index += 1
+            return self._data[self._index-1]
+        raise StopAsyncIteration
+
+class Corotest(object):
+    def __init__(self):
+        self.res = "-"
+    
+    async def do_loop(self):
+        async for x in AsyncIter():
+            self.res += str(x)
+            self.res += "-"
+
+cor = Corotest()
+loop = asyncio.get_event_loop()
+futures = [asyncio.ensure_future(cor.do_loop()), asyncio.ensure_future(cor.do_loop())]
+taskres = loop.run_until_complete(asyncio.wait(futures))
+assert cor.res.count('0') == 2
+assert cor.res.count('1') == 2
+assert cor.res.count('2') == 2
+assert cor.res.count('3') == 2
+assert cor.res.count('4') == 2
+assert cor.res.find("0") < cor.res.find("1")
+assert cor.res.find("1") < cor.res.find("2")
+assert cor.res.find("2") < cor.res.find("3")
+assert cor.res.find("3") < cor.res.find("4")
+assert isinstance(taskres, tuple)
+assert len(taskres) == 2
+assert "result=None" in repr(taskres[0].pop())
+assert "result=None" in repr(taskres[0].pop())
         """
     
     def test_asynchronous_context_managers(self):
+        # it is important that "releasing lock A" happens before "holding lock B"
+        # or the other way around, but it is not allowed that both coroutines
+        # hold the lock at the same time
         """
 import encodings.idna
 import asyncio
@@ -44,5 +99,12 @@
 finally:
     loop.close()
 
-assert cor.res == "- coro 1: waiting for lock - coro 1: holding the lock - coro 2: waiting for lock - coro 1: releasing the lock - coro 2: holding the lock - coro 2: releasing the lock -"
+assert "coro 1: waiting for lock" in cor.res
+assert "coro 1: holding the lock" in cor.res
+assert "coro 1: releasing the lock" in cor.res
+assert "coro 2: waiting for lock" in cor.res
+assert "coro 2: holding the lock" in cor.res
+assert "coro 2: releasing the lock" in cor.res
+assert cor.res.find("coro 1: releasing the lock") < cor.res.find("coro 2: holding the lock") or \
+cor.res.find("coro 2: releasing the lock") < cor.res.find("coro 1: holding the lock")
         """
diff --git a/pypy/module/exceptions/__init__.py b/pypy/module/exceptions/__init__.py
--- a/pypy/module/exceptions/__init__.py
+++ b/pypy/module/exceptions/__init__.py
@@ -52,6 +52,7 @@
         'ResourceWarning'  : 'interp_exceptions.W_ResourceWarning',
         'RuntimeError' : 'interp_exceptions.W_RuntimeError',
         'RuntimeWarning' : 'interp_exceptions.W_RuntimeWarning',
+        'StopAsyncIteration' : 'interp_exceptions.W_StopAsyncIteration',
         'StopIteration' : 'interp_exceptions.W_StopIteration',
         'SyntaxError' : 'interp_exceptions.W_SyntaxError',
         'SyntaxWarning' : 'interp_exceptions.W_SyntaxWarning',
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -65,6 +65,7 @@
       +-- RuntimeError
       |    +-- NotImplementedError
       |    +-- RecursionError
+      +-- StopAsyncIteration
       +-- SyntaxError
       |    +-- IndentationError
       |         +-- TabError
@@ -834,6 +835,9 @@
 W_AssertionError = _new_exception('AssertionError', W_Exception,
                                   """Assertion failed.""")
 
+W_StopAsyncIteration = _new_exception('StopAsyncIteration', W_Exception,
+                                  """Signal the end from iterator.__anext__().""")
+
 class W_UnicodeDecodeError(W_UnicodeError):
     """Unicode decoding error."""
     w_encoding = None
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -298,7 +298,10 @@
         return w_iter
 
     def next(space, w_obj):
-        w_descr = space.lookup(w_obj, '__next__')
+        if space.type(w_obj).name == 'coroutine':
+            w_descr = space.lookup(w_obj, '__anext__')
+        else:
+            w_descr = space.lookup(w_obj, '__next__')
         if w_descr is None:
             raise oefmt(space.w_TypeError,
                         "'%T' object is not an iterator", w_obj)

From pypy.commits at gmail.com  Thu Aug 18 14:45:05 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 18 Aug 2016 11:45:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Change POP_EXCEPT back to -1
Message-ID: <57b60231.8411c20a.a530c.2ce4@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86289:6cf3401c7f36
Date: 2016-08-18 20:44 +0200
http://bitbucket.org/pypy/pypy/changeset/6cf3401c7f36/

Log:	Change POP_EXCEPT back to -1

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -609,7 +609,7 @@
     ops.WITH_CLEANUP_FINISH: -2,
     ops.LOAD_BUILD_CLASS: 1,
     ops.POP_BLOCK: 0,
-    ops.POP_EXCEPT: -2,
+    ops.POP_EXCEPT: -1,
     ops.END_FINALLY: -4,     # assume always 4: we pretend that SETUP_FINALLY
                              # pushes 4.  In truth, it would only push 1 and
                              # the corresponding END_FINALLY only pops 1.

From pypy.commits at gmail.com  Thu Aug 18 15:34:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 12:34:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Very obscure fix
Message-ID: <57b60dde.11051c0a.d4d95.5ce7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86290:4b361d9f44e2
Date: 2016-08-18 21:34 +0200
http://bitbucket.org/pypy/pypy/changeset/4b361d9f44e2/

Log:	Very obscure fix

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -632,6 +632,13 @@
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_EXCEPT) # for SETUP_EXCEPT
+        # Manually remove the 'aiter' object from the valuestack.
+        # This POP_TOP is not needed from the point of view of
+        # pyopcode.py, which will pop anything to match the stack
+        # depth of the SETUP_LOOP, but it is needed to make
+        # PythonCodeMaker._stacksize() compute an exact result and not
+        # crash with StackDepthComputationError.
+        self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP
         self.emit_jump(ops.JUMP_ABSOLUTE, b_after_loop_else, True)
         

From pypy.commits at gmail.com  Thu Aug 18 15:56:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 12:56:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Describe more clearly that the (first)
 StackDepthComputationError can be
Message-ID: <57b612e3.531d1c0a.504c5.61be@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86291:e0ca92dd6b05
Date: 2016-08-18 21:55 +0200
http://bitbucket.org/pypy/pypy/changeset/e0ca92dd6b05/

Log:	Describe more clearly that the (first) StackDepthComputationError
	can be ignored at first.

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -398,9 +398,17 @@
         for block in blocks:
             depth = self._do_stack_depth_walk(block)
             if block.auto_inserted_return and depth != 0:
-                os.write(2, "StackDepthComputationError in %s at %s:%s\n" % (
-                    self.compile_info.filename, self.name, self.first_lineno))
-                raise StackDepthComputationError   # fatal error
+                # This case occurs if this code object uses some
+                # construction for which the stack depth computation
+                # is wrong (too high).  If you get here while working
+                # on the astcompiler, then you should at first ignore
+                # the error, and comment out the 'raise' below.  Such
+                # an error is not really bad: it is just a bit
+                # wasteful.  For release-ready versions, though, we'd
+                # like not to be wasteful. :-)
+                os.write(2, "StackDepthComputationError(POS) in %s at %s:%s\n"
+                  % (self.compile_info.filename, self.name, self.first_lineno))
+                raise StackDepthComputationError   # would-be-nice-not-to-have
         return self._max_depth
 
     def _next_stack_depth_walk(self, nextblock, depth):
@@ -413,7 +421,16 @@
              return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
-            assert depth >= 0
+            if depth < 0:
+                # This is really a fatal error, don't comment out this
+                # 'raise'.  It means that the stack depth computation
+                # thinks there is a path that yields a negative stack
+                # depth, which means that it underestimates the space
+                # needed and it would crash when interpreting this
+                # code.
+                os.write(2, "StackDepthComputationError(NEG) in %s at %s:%s\n"
+                  % (self.compile_info.filename, self.name, self.first_lineno))
+                raise StackDepthComputationError   # really fatal error
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode

From pypy.commits at gmail.com  Thu Aug 18 16:11:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 13:11:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: Clean up Signature to stop making it
 "tuply". It's now a lie that it is
Message-ID: <57b6168a.c19d1c0a.a7ff3.6906@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86292:49dbf1f24170
Date: 2016-08-18 22:11 +0200
http://bitbucket.org/pypy/pypy/changeset/49dbf1f24170/

Log:	Clean up Signature to stop making it "tuply". It's now a lie that it
	is used by the annotator.

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -596,7 +596,10 @@
 
         # First extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(func.func_code)
+        sig = pycode.cpython_code_signature(func.func_code)
+        argnames = sig.argnames
+        varargname = sig.varargname
+        kwargname = sig.kwargname
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -620,7 +623,9 @@
         app_sig = SignatureBuilder(func)
 
         UnwrapSpec_Check(orig_sig).apply_over(unwrap_spec, app_sig)
-        self.sig = argnames, varargname, kwargname = app_sig.signature()
+        self.sig = app_sig.signature()
+        argnames = self.sig.argnames
+        varargname = self.sig.varargname
 
         self.minargs = len(argnames)
         if varargname:
@@ -951,7 +956,7 @@
                 defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
             UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig[0])
+            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
             if defs_w:
                 alldefs_w[-len(defs_w):] = defs_w
             code = self._code
@@ -968,7 +973,7 @@
                     assert isinstance(w_default, W_Root)
                     assert argname.startswith('w_')
                     argname = argname[2:]
-                    j = self._code.sig[0].index(argname)
+                    j = self._code.sig.argnames.index(argname)
                     assert alldefs_w[j] in (UNDEFINED, None)
                     alldefs_w[j] = w_default
             first_defined = 0
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -37,7 +37,7 @@
 
 # cpython_code_signature helper
 def cpython_code_signature(code):
-    "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
+    """Return a Signature instance."""
     argcount = code.co_argcount
     varnames = code.co_varnames
     assert argcount >= 0     # annotator hint
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -55,18 +55,3 @@
         if not isinstance(other, Signature):
             return NotImplemented
         return not self == other
-
-
-    # make it look tuply for its use in the annotator
-
-    def __len__(self):
-        return 3
-
-    def __getitem__(self, i):
-        if i == 0:
-            return self.argnames
-        if i == 1:
-            return self.varargname
-        if i == 2:
-            return self.kwargname
-        raise IndexError
\ No newline at end of file
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -46,13 +46,6 @@
         assert sig.find_argname("c") == 2
         assert sig.find_argname("d") == -1
 
-    def test_tuply(self):
-        sig = Signature(["a", "b", "c"], "d", "e")
-        x, y, z = sig
-        assert x == ["a", "b", "c"]
-        assert y == "d"
-        assert z == "e"
-
 class dummy_wrapped_dict(dict):
     def __nonzero__(self):
         raise NotImplementedError
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -203,7 +203,8 @@
     name = func.__name__
     extra = ', '.join(extraargs)
     from pypy.interpreter import pycode
-    argnames, _, _ = pycode.cpython_code_signature(func.func_code)
+    sig = pycode.cpython_code_signature(func.func_code)
+    argnames = sig.argnames
     if use_closure:
         if argnames[1] == 'space':
             args = "closure, space, obj"

From pypy.commits at gmail.com  Thu Aug 18 16:14:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 13:14:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57b61722.28eac20a.e7e57.4af5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86293:b34a037a6866
Date: 2016-08-18 22:12 +0200
http://bitbucket.org/pypy/pypy/changeset/b34a037a6866/

Log:	hg merge default

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -389,7 +389,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -408,8 +409,11 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            assert depth >= 0
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -612,7 +612,10 @@
 
         # First extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(func.func_code)
+        sig = pycode.cpython_code_signature(func.func_code)
+        argnames = sig.argnames
+        varargname = sig.varargname
+        kwargname = sig.kwargname
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -636,7 +639,9 @@
         app_sig = SignatureBuilder(func)
 
         UnwrapSpec_Check(orig_sig).apply_over(unwrap_spec, app_sig)
-        self.sig = argnames, varargname, kwargname = app_sig.signature()
+        self.sig = app_sig.signature()
+        argnames = self.sig.argnames
+        varargname = self.sig.varargname
 
         self.minargs = len(argnames)
         if varargname:
@@ -972,7 +977,7 @@
                     defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
             UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig[0])
+            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
             if defs_w:
                 alldefs_w[-len(defs_w):] = defs_w
             code = self._code
@@ -993,7 +998,7 @@
                     assert isinstance(w_default, W_Root)
                     assert argname.startswith('w_')
                     argname = argname[2:]
-                    j = self._code.sig[0].index(argname)
+                    j = self._code.sig.argnames.index(argname)
                     assert alldefs_w[j] in (UNDEFINED, None)
                     alldefs_w[j] = w_default
             first_defined = 0
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -45,7 +45,7 @@
 
 # cpython_code_signature helper
 def cpython_code_signature(code):
-    "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
+    """Return a Signature instance."""
     argcount = code.co_argcount
     varnames = code.co_varnames
     if we_are_translated():
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -68,18 +68,3 @@
         if not isinstance(other, Signature):
             return NotImplemented
         return not self == other
-
-
-    # make it look tuply for its use in the annotator
-
-    def __len__(self):
-        return 3
-
-    def __getitem__(self, i):
-        if i == 0:
-            return self.argnames
-        if i == 1:
-            return self.varargname
-        if i == 2:
-            return self.kwargname
-        raise IndexError
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -47,13 +47,6 @@
         assert sig.find_argname("d") == -1
         assert sig.find_argname("kwonly") == 3
 
-    def test_tuply(self):
-        sig = Signature(["a", "b", "c"], "d", "e")
-        x, y, z = sig
-        assert x == ["a", "b", "c"]
-        assert y == "d"
-        assert z == "e"
-
 class dummy_wrapped_dict(dict):
     def __nonzero__(self):
         raise NotImplementedError
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -202,7 +202,8 @@
     name = func.__name__
     extra = ', '.join(extraargs)
     from pypy.interpreter import pycode
-    argnames, _, _ = pycode.cpython_code_signature(func.func_code)
+    sig = pycode.cpython_code_signature(func.func_code)
+    argnames = sig.argnames
     if use_closure:
         if argnames[1] == 'space':
             args = "closure, space, obj"
diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -10,10 +10,10 @@
 
     def setup_class(cls):
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__jitlog.1')))
-        cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
-        cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
+        cls.w_mark_header = cls.space.newbytes(jl.MARK_JITLOG_HEADER)
+        cls.w_version = cls.space.newbytes(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
-        cls.w_machine = cls.space.wrap(platform.machine())
+        cls.w_machine = cls.space.newbytes(platform.machine())
         cls.w_resops = cls.space.newdict()
         space = cls.space
         for key, value in opname.items():
@@ -48,5 +48,3 @@
                 assert opnum in self.resops
                 # the name must equal
                 assert self.resops[opnum] == opname
-
-
diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py
--- a/pypy/tool/pytest/genreportdata.py
+++ b/pypy/tool/pytest/genreportdata.py
@@ -17,7 +17,7 @@
             resultwc = py.path.svnwc(testresultdir)
             print "updating", resultwc
             resultwc.update()
-        except KeyboardInterrupt as RuntimeError:
+        except (KeyboardInterrupt, RuntimeError):
             raise
         except Exception as e: #py.process.ExecutionFailed,e:
             print >> sys.stderr, "Warning: ",e #Subversion update failed"

From pypy.commits at gmail.com  Thu Aug 18 16:14:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 13:14:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57b61723.8f8e1c0a.2f009.6591@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86294:3c9a0b0aa820
Date: 2016-08-18 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/3c9a0b0aa820/

Log:	hg merge py3k

diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -118,7 +118,7 @@
     # don't constant-fold if "w_left" and "w_right" are integers and
     # the estimated bit length of the power is unreasonably large
     space.appexec([w_left, w_right], """(left, right):
-        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+        if isinstance(left, int) and isinstance(right, int):
             if left.bit_length() * right > 5000:
                 raise OverflowError
     """)
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -612,7 +612,10 @@
 
         # First extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(func.func_code)
+        sig = pycode.cpython_code_signature(func.func_code)
+        argnames = sig.argnames
+        varargname = sig.varargname
+        kwargname = sig.kwargname
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -636,7 +639,9 @@
         app_sig = SignatureBuilder(func)
 
         UnwrapSpec_Check(orig_sig).apply_over(unwrap_spec, app_sig)
-        self.sig = argnames, varargname, kwargname = app_sig.signature()
+        self.sig = app_sig.signature()
+        argnames = self.sig.argnames
+        varargname = self.sig.varargname
 
         self.minargs = len(argnames)
         if varargname:
@@ -972,7 +977,7 @@
                     defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
             UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig[0])
+            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
             if defs_w:
                 alldefs_w[-len(defs_w):] = defs_w
             code = self._code
@@ -993,7 +998,7 @@
                     assert isinstance(w_default, W_Root)
                     assert argname.startswith('w_')
                     argname = argname[2:]
-                    j = self._code.sig[0].index(argname)
+                    j = self._code.sig.argnames.index(argname)
                     assert alldefs_w[j] in (UNDEFINED, None)
                     alldefs_w[j] = w_default
             first_defined = 0
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -45,7 +45,7 @@
 
 # cpython_code_signature helper
 def cpython_code_signature(code):
-    "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
+    """Return a Signature instance."""
     argcount = code.co_argcount
     varnames = code.co_varnames
     if we_are_translated():
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -68,18 +68,3 @@
         if not isinstance(other, Signature):
             return NotImplemented
         return not self == other
-
-
-    # make it look tuply for its use in the annotator
-
-    def __len__(self):
-        return 3
-
-    def __getitem__(self, i):
-        if i == 0:
-            return self.argnames
-        if i == 1:
-            return self.varargname
-        if i == 2:
-            return self.kwargname
-        raise IndexError
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -47,13 +47,6 @@
         assert sig.find_argname("d") == -1
         assert sig.find_argname("kwonly") == 3
 
-    def test_tuply(self):
-        sig = Signature(["a", "b", "c"], "d", "e")
-        x, y, z = sig
-        assert x == ["a", "b", "c"]
-        assert y == "d"
-        assert z == "e"
-
 class dummy_wrapped_dict(dict):
     def __nonzero__(self):
         raise NotImplementedError
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -202,7 +202,8 @@
     name = func.__name__
     extra = ', '.join(extraargs)
     from pypy.interpreter import pycode
-    argnames, _, _ = pycode.cpython_code_signature(func.func_code)
+    sig = pycode.cpython_code_signature(func.func_code)
+    argnames = sig.argnames
     if use_closure:
         if argnames[1] == 'space':
             args = "closure, space, obj"
diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -10,10 +10,10 @@
 
     def setup_class(cls):
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__jitlog.1')))
-        cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
-        cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
+        cls.w_mark_header = cls.space.newbytes(jl.MARK_JITLOG_HEADER)
+        cls.w_version = cls.space.newbytes(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
-        cls.w_machine = cls.space.wrap(platform.machine())
+        cls.w_machine = cls.space.newbytes(platform.machine())
         cls.w_resops = cls.space.newdict()
         space = cls.space
         for key, value in opname.items():
@@ -48,5 +48,3 @@
                 assert opnum in self.resops
                 # the name must equal
                 assert self.resops[opnum] == opname
-
-
diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py
--- a/pypy/tool/pytest/genreportdata.py
+++ b/pypy/tool/pytest/genreportdata.py
@@ -17,7 +17,7 @@
             resultwc = py.path.svnwc(testresultdir)
             print "updating", resultwc
             resultwc.update()
-        except KeyboardInterrupt as RuntimeError:
+        except (KeyboardInterrupt, RuntimeError):
             raise
         except Exception as e: #py.process.ExecutionFailed,e:
             print >> sys.stderr, "Warning: ",e #Subversion update failed"

From pypy.commits at gmail.com  Thu Aug 18 17:30:01 2016
From: pypy.commits at gmail.com (wlav)
Date: Thu, 18 Aug 2016 14:30:01 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: fix string handling for the
 loadable capi
Message-ID: <57b628d9.497bc20a.7638e.60e1@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86295:4831e0f5b4d5
Date: 2016-08-18 13:41 -0700
http://bitbucket.org/pypy/pypy/changeset/4831e0f5b4d5/

Log:	fix string handling for the loadable capi

diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -354,7 +354,7 @@
         i += 1
         py_indices.append(index)
         index = indices[i]
-    c_free(rffi.cast(rffi.VOIDP, indices))   # c_free defined below
+    c_free(space, rffi.cast(rffi.VOIDP, indices))      # c_free defined below
     return py_indices
 
 _c_method_name = rffi.llexternal(
@@ -533,16 +533,18 @@
     compilation_info=backend.eci)
 def c_strtoull(space, svalue):
     return _c_strtoull(svalue)
-c_free = rffi.llexternal(
+_c_free = rffi.llexternal(
     "cppyy_free",
     [rffi.VOIDP], lltype.Void,
     releasegil=ts_memory,
     compilation_info=backend.eci)
+def c_free(space, voidp):
+    return _c_free(voidp)
 
 def charp2str_free(space, charp):
     string = rffi.charp2str(charp)
     voidp = rffi.cast(rffi.VOIDP, charp)
-    c_free(voidp)
+    _c_free(voidp)
     return string
 
 _c_charp2stdstring = rffi.llexternal(
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -220,7 +220,7 @@
 
             'charp2stdstring'          : ([c_ccharp, c_size_t],       c_object),
             #stdstring2charp  actually takes an size_t* as last parameter, but this will do
-            'stdstring2charp'          : ([c_ccharp, c_voidp],        c_ccharp),
+            'stdstring2charp'          : ([c_object, c_voidp],        c_ccharp),
             'stdstring2stdstring'      : ([c_object],                 c_object),
         }
 
@@ -281,6 +281,10 @@
     ptr = w_cdata.unsafe_escaping_ptr()
     return rffi.cast(rffi.VOIDP, ptr)
 
+def _cdata_to_ccharp(space, w_cdata):
+    ptr = _cdata_to_ptr(space, w_cdata)      # see above ... something better?
+    return rffi.cast(rffi.CCHARP, ptr)
+
 def c_load_dictionary(name):
     return libffi.CDLL(name)
 
@@ -341,11 +345,14 @@
     return _cdata_to_ptr(space, call_capi(space, 'call_r', args))
 def c_call_s(space, cppmethod, cppobject, nargs, cargs):
     length = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs), _Arg(vp=length)]
-    cstr = call_capi(space, 'call_s', args)
-    cstr_len = int(length[0])
-    lltype.free(length, flavor='raw')
-    return cstr, cstr_len
+    try:
+        w_cstr = call_capi(space, 'call_s',
+            [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs),
+             _Arg(vp=rffi.cast(rffi.VOIDP, length))])
+        cstr_len = int(length[0])
+    finally:
+        lltype.free(length, flavor='raw')
+    return _cdata_to_ccharp(space, w_cstr), cstr_len
 
 def c_constructor(space, cppmethod, cppobject, nargs, cargs):
     args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
@@ -527,15 +534,16 @@
 
 def c_charp2stdstring(space, svalue, sz):
     return _cdata_to_cobject(
-        space, call_capi(space, 'charp2stdstring', [_Arg(s=svalue), _Arg(l=sz)]))
+        space, call_capi(space, 'charp2stdstring', [_Arg(s=svalue), _Arg(h=sz)]))
 def c_stdstring2charp(space, cppstr):
     sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
     try:
-        cstr = call_capi(space, 'stdstring2charp', [_Arg(h=cppstr), _Arg(vp=sz)])
+        w_cstr = call_capi(space, 'stdstring2charp',
+            [_Arg(h=cppstr), _Arg(vp=rffi.cast(rffi.VOIDP, sz))])
         cstr_len = int(sz[0])
     finally:
         lltype.free(sz, flavor='raw')
-    return rffi.charpsize2str(cstr, cstr_len)
+    return rffi.charpsize2str(_cdata_to_ccharp(space, w_cstr), cstr_len)
 def c_stdstring2stdstring(space, cppobject):
     return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_Arg(h=cppobject)]))
 
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -196,9 +196,9 @@
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         cstr, cstr_len = capi.c_call_s(space, cppmethod, cppthis, num_args, args)
-        string = rffi.charpsize2str(cstr, cstr_len)
-        capi.c_free(rffi.cast(rffi.VOIDP, cstr))
-        return space.wrap(string)
+        pystr = rffi.charpsize2str(cstr, cstr_len)
+        capi.c_free(space, rffi.cast(rffi.VOIDP, cstr))
+        return space.wrap(pystr)
 
     def execute_libffi(self, space, cif_descr, funcaddr, buffer):
         from pypy.module.cppyy.interp_cppyy import FastCallNotPossible

From pypy.commits at gmail.com  Thu Aug 18 17:30:03 2016
From: pypy.commits at gmail.com (wlav)
Date: Thu, 18 Aug 2016 14:30:03 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: change handling of std;
 fixes access to cout
Message-ID: <57b628db.c19d1c0a.a7ff3.81fb@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86296:1eaa1b5ee34f
Date: 2016-08-18 14:17 -0700
http://bitbucket.org/pypy/pypy/changeset/1eaa1b5ee34f/

Log:	change handling of std; fixes access to cout

diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -440,7 +440,7 @@
     gbl = make_cppnamespace(None, "::", None, False)   # global C++ namespace
     gbl.__doc__ = "Global C++ namespace."
 
-    # mostly for the benefit of the CINT backend, which treats std as special
+    # pre-create std to allow direct importing
     gbl.std = make_cppnamespace(None, "std", None, False)
 
     # install a type for enums to refer to
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -70,9 +70,10 @@
       assert( g_classrefs.size() == GLOBAL_HANDLE );
       g_name2classrefidx[ "" ]      = GLOBAL_HANDLE;
       g_classrefs.push_back(TClassRef(""));
-      // ROOT ignores std/::std, so point them to the global namespace
-      g_name2classrefidx[ "std" ]   = GLOBAL_HANDLE;
-      g_name2classrefidx[ "::std" ] = GLOBAL_HANDLE;
+      // aliases for std (setup already in pythonify)
+      g_name2classrefidx[ "std" ]   = GLOBAL_HANDLE+1;
+      g_name2classrefidx[ "::std" ] = GLOBAL_HANDLE+1;
+      g_classrefs.push_back(TClassRef("std"));
       // add a dummy global to refer to as null at index 0
       g_globalvars.push_back( nullptr );
    }
@@ -1012,8 +1013,10 @@
 
 Cppyy::TCppIndex_t Cppyy::GetDatamemberIndex( TCppScope_t scope, const std::string& name )
 {
+   std::cout << " ASKING FOR: " << name << " on scope: " << scope << std::endl;
    if ( scope == GLOBAL_HANDLE ) {
       TGlobal* gb = (TGlobal*)gROOT->GetListOfGlobals( kTRUE )->FindObject( name.c_str() );
+      std::cout << " FOUND (G): "<< gb << " " << (TGlobal*)gROOT->GetListOfGlobals( kTRUE )->FindObject("std::cout") << std::endl;
       if ( gb && gb->GetAddress() && gb->GetAddress() != (void*)-1 ) {
          g_globalvars.push_back( gb );
          return g_globalvars.size() - 1;
@@ -1025,6 +1028,7 @@
          TDataMember* dm =
             (TDataMember*)cr->GetListOfDataMembers()->FindObject( name.c_str() );
          // TODO: turning this into an index is silly ...
+         std::cout << " FOUND (D): "<< dm << std::endl;
          if ( dm ) return (TCppIndex_t)cr->GetListOfDataMembers()->IndexOf( dm );
       }
    }
diff --git a/pypy/module/cppyy/test/Makefile b/pypy/module/cppyy/test/Makefile
--- a/pypy/module/cppyy/test/Makefile
+++ b/pypy/module/cppyy/test/Makefile
@@ -62,14 +62,12 @@
 
 endif
 
-ifeq ($(CLING),)
 ifeq ($(DUMMY),)
 # TODO: methptrgetter causes these tests to crash, so don't use it for now
 std_streamsDict.so: std_streams.cxx std_streams.h std_streams.xml
 	$(genreflex) std_streams.h --selection=std_streams.xml
 	g++ -o $@ std_streams_rflx.cpp std_streams.cxx -shared -std=c++14 $(cppflags) $(cppflags2)
 endif
-endif
 
 .PHONY: clean
 clean:
diff --git a/pypy/module/cppyy/test/std_streams.xml b/pypy/module/cppyy/test/std_streams.xml
--- a/pypy/module/cppyy/test/std_streams.xml
+++ b/pypy/module/cppyy/test/std_streams.xml
@@ -1,9 +1,12 @@
 <lcgdict>
 
+  <namespace name = "std" />
+
   <class name = "std::ostream"/>
   <class name = "std::ios_base"/>
   <class name = "std::basic_ios<char,std::char_traits<char> >"/>
 
+  <class name = "std::basic_ostream<char,char_traits<char> >" />
   <variable name="std::cout" />
 
 </lcgdict>

From pypy.commits at gmail.com  Thu Aug 18 18:38:51 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 Aug 2016 15:38:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix missing change for the modified
 Signature API in 49dbf1f24170
Message-ID: <57b638fb.0205c20a.deaff.74fd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86297:ef1ddb6992ab
Date: 2016-08-18 23:38 +0100
http://bitbucket.org/pypy/pypy/changeset/ef1ddb6992ab/

Log:	Fix missing change for the modified Signature API in 49dbf1f24170

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,7 +11,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
-from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite,
         c_fdopen, c_fileno,
         c_fopen)# for tests
 from rpython.translator import cdir
@@ -259,14 +259,14 @@
 
         # extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(callable.func_code)
+        sig = pycode.cpython_code_signature(callable.func_code)
+        assert sig.argnames[0] == 'space'
+        self.argnames = sig.argnames[1:]
+        if gil == 'pygilstate_ensure':
+            assert self.argnames[-1] == 'previous_state'
+            del self.argnames[-1]
+        assert len(self.argnames) == len(self.argtypes)
 
-        assert argnames[0] == 'space'
-        if gil == 'pygilstate_ensure':
-            assert argnames[-1] == 'previous_state'
-            del argnames[-1]
-        self.argnames = argnames[1:]
-        assert len(self.argnames) == len(self.argtypes)
         self.gil = gil
         self.result_borrowed = result_borrowed
         self.result_is_ll = result_is_ll

From pypy.commits at gmail.com  Thu Aug 18 18:40:11 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 Aug 2016 15:40:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57b6394b.04141c0a.e63b0.944d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86298:71a1d16e20d5
Date: 2016-08-18 23:39 +0100
http://bitbucket.org/pypy/pypy/changeset/71a1d16e20d5/

Log:	hg merge default

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,7 +11,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
-from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite,
         c_fdopen, c_fileno,
         c_fopen)# for tests
 from rpython.translator import cdir
@@ -259,14 +259,14 @@
 
         # extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(callable.func_code)
+        sig = pycode.cpython_code_signature(callable.func_code)
+        assert sig.argnames[0] == 'space'
+        self.argnames = sig.argnames[1:]
+        if gil == 'pygilstate_ensure':
+            assert self.argnames[-1] == 'previous_state'
+            del self.argnames[-1]
+        assert len(self.argnames) == len(self.argtypes)
 
-        assert argnames[0] == 'space'
-        if gil == 'pygilstate_ensure':
-            assert argnames[-1] == 'previous_state'
-            del argnames[-1]
-        self.argnames = argnames[1:]
-        assert len(self.argnames) == len(self.argtypes)
         self.gil = gil
         self.result_borrowed = result_borrowed
         self.result_is_ll = result_is_ll

From pypy.commits at gmail.com  Thu Aug 18 18:40:41 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 Aug 2016 15:40:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57b63969.a710c20a.b630c.762e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r86299:1dd495a13d2c
Date: 2016-08-18 23:39 +0100
http://bitbucket.org/pypy/pypy/changeset/1dd495a13d2c/

Log:	hg merge py3k

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,7 +11,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
-from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite,
         c_fdopen, c_fileno,
         c_fopen)# for tests
 from rpython.translator import cdir
@@ -259,14 +259,14 @@
 
         # extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(callable.func_code)
+        sig = pycode.cpython_code_signature(callable.func_code)
+        assert sig.argnames[0] == 'space'
+        self.argnames = sig.argnames[1:]
+        if gil == 'pygilstate_ensure':
+            assert self.argnames[-1] == 'previous_state'
+            del self.argnames[-1]
+        assert len(self.argnames) == len(self.argtypes)
 
-        assert argnames[0] == 'space'
-        if gil == 'pygilstate_ensure':
-            assert argnames[-1] == 'previous_state'
-            del argnames[-1]
-        self.argnames = argnames[1:]
-        assert len(self.argnames) == len(self.argtypes)
         self.gil = gil
         self.result_borrowed = result_borrowed
         self.result_is_ll = result_is_ll

From pypy.commits at gmail.com  Thu Aug 18 19:03:16 2016
From: pypy.commits at gmail.com (wlav)
Date: Thu, 18 Aug 2016 16:03:16 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: remove spurious printout
Message-ID: <57b63eb4.e129c20a.f2ff2.7ba0@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86300:4ea4f34d66b9
Date: 2016-08-18 14:52 -0700
http://bitbucket.org/pypy/pypy/changeset/4ea4f34d66b9/

Log:	remove spurious printout

diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -1013,10 +1013,8 @@
 
 Cppyy::TCppIndex_t Cppyy::GetDatamemberIndex( TCppScope_t scope, const std::string& name )
 {
-   std::cout << " ASKING FOR: " << name << " on scope: " << scope << std::endl;
    if ( scope == GLOBAL_HANDLE ) {
       TGlobal* gb = (TGlobal*)gROOT->GetListOfGlobals( kTRUE )->FindObject( name.c_str() );
-      std::cout << " FOUND (G): "<< gb << " " << (TGlobal*)gROOT->GetListOfGlobals( kTRUE )->FindObject("std::cout") << std::endl;
       if ( gb && gb->GetAddress() && gb->GetAddress() != (void*)-1 ) {
          g_globalvars.push_back( gb );
          return g_globalvars.size() - 1;
@@ -1028,7 +1026,6 @@
          TDataMember* dm =
             (TDataMember*)cr->GetListOfDataMembers()->FindObject( name.c_str() );
          // TODO: turning this into an index is silly ...
-         std::cout << " FOUND (D): "<< dm << std::endl;
          if ( dm ) return (TCppIndex_t)cr->GetListOfDataMembers()->IndexOf( dm );
       }
    }

From pypy.commits at gmail.com  Thu Aug 18 19:03:17 2016
From: pypy.commits at gmail.com (wlav)
Date: Thu, 18 Aug 2016 16:03:17 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: translater fixes
Message-ID: <57b63eb5.151a1c0a.5dc6.9a22@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86301:e2e2d5f25c36
Date: 2016-08-18 15:59 -0700
http://bitbucket.org/pypy/pypy/changeset/e2e2d5f25c36/

Log:	translater fixes

diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -1,4 +1,5 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import jit
 
 import cling_capi as backend
@@ -165,7 +166,7 @@
     length = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
     try:
         cstr = _c_call_s(cppmethod, cppobject, nargs, args, length)
-        cstr_len = int(length[0])
+        cstr_len = intmask(length[0])
     finally:
         lltype.free(length, flavor='raw')
     return cstr, cstr_len
diff --git a/pypy/module/cppyy/capi/cling_capi.py b/pypy/module/cppyy/capi/cling_capi.py
--- a/pypy/module/cppyy/capi/cling_capi.py
+++ b/pypy/module/cppyy/capi/cling_capi.py
@@ -4,6 +4,7 @@
 
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import libffi, rdynload
 
 from pypy.module.cppyy.capi.capi_types import C_OBJECT
@@ -81,7 +82,7 @@
     sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
     try:
         cstr = _c_stdstring2charp(cppstr, sz)
-        cstr_len = int(sz[0])
+        cstr_len = intmask(sz[0])
     finally:
         lltype.free(sz, flavor='raw')
     return rffi.charpsize2str(cstr, cstr_len)
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -1,4 +1,5 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import jit, jit_libffi, libffi, rdynload, objectmodel
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.tool import leakfinder
@@ -349,7 +350,7 @@
         w_cstr = call_capi(space, 'call_s',
             [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs),
              _Arg(vp=rffi.cast(rffi.VOIDP, length))])
-        cstr_len = int(length[0])
+        cstr_len = intmask(length[0])
     finally:
         lltype.free(length, flavor='raw')
     return _cdata_to_ccharp(space, w_cstr), cstr_len
@@ -540,7 +541,7 @@
     try:
         w_cstr = call_capi(space, 'stdstring2charp',
             [_Arg(h=cppstr), _Arg(vp=rffi.cast(rffi.VOIDP, sz))])
-        cstr_len = int(sz[0])
+        cstr_len = intmask(sz[0])
     finally:
         lltype.free(sz, flavor='raw')
     return rffi.charpsize2str(_cdata_to_ccharp(space, w_cstr), cstr_len)

From pypy.commits at gmail.com  Fri Aug 19 01:41:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 22:41:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backed out changeset 33d4dcf474ce
Message-ID: <57b69c13.a710c20a.b630c.c8a1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86302:916d5087f471
Date: 2016-08-19 07:41 +0200
http://bitbucket.org/pypy/pypy/changeset/916d5087f471/

Log:	Backed out changeset 33d4dcf474ce

	This broke tests (just run py.test test_re_python.py). We were
	importing modules in every single test, and teardown_method() was
	cleaning it up, to check that there would be no leak.

diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,21 +69,10 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        cls.w_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
+        space.appexec([space.wrap(str(tmpdir))], """(path):
             import _cffi_backend     # force it to be initialized
             import sys
             sys.path.insert(0, path)
-            from re_python_pysrc import ffi
-            del sys.path[0]
-            return ffi
-        """)
-        cls.w_sub_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
-            from re_py_subsrc import ffi
-            del sys.path[0]
-            return ffi
         """)
 
     def teardown_method(self, meth):
@@ -97,25 +86,25 @@
 
 
     def test_constant_1(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
         assert type(lib.add42) is _cffi_backend.FFI.CData
 
     def test_dlclose(self):
         import _cffi_backend
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
         e = raises(ffi.error, ffi.dlclose, lib)
@@ -126,18 +115,18 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
             assert p.x == 5
@@ -145,13 +134,13 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
-        ffi = self.sub_ffi
+        from re_py_subsrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
         assert ffi.integer_const('k2') == 121212
@@ -164,7 +153,7 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
         p = ffi.addressof(lib, 'globalvar42')
@@ -174,25 +163,25 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == "hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
-        ffi = self.ffi
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
         assert str(e.value).startswith(

From pypy.commits at gmail.com  Fri Aug 19 01:54:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 22:54:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: Skip these two tests if running on a
 very old 2.7, where at least on
Message-ID: <57b69f24.a699c20a.b61fd.d1b6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86303:c02fee85a03a
Date: 2016-08-19 07:54 +0200
http://bitbucket.org/pypy/pypy/changeset/c02fee85a03a/

Log:	Skip these two tests if running on a very old 2.7, where at least on
	32-bit we get OverflowErrors inside rlib/rposix.py

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -619,6 +619,7 @@
             assert os.geteuid() == self.geteuid
 
     if hasattr(os, 'setuid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setuid_error(self):
             os = self.posix
             raises(OverflowError, os.setuid, -2)
@@ -666,6 +667,7 @@
             raises(OSError, os.getpgid, 1234567)
 
     if hasattr(os, 'setgid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setgid_error(self):
             os = self.posix
             raises(OverflowError, os.setgid, -2)

From pypy.commits at gmail.com  Fri Aug 19 02:09:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 Aug 2016 23:09:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix the original problem differently
Message-ID: <57b6a2a4.4abf1c0a.b6879.f1de@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86304:2430635c051f
Date: 2016-08-19 08:09 +0200
http://bitbucket.org/pypy/pypy/changeset/2430635c051f/

Log:	Fix the original problem differently

diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,10 +69,13 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
+        cls.w_fix_path = space.appexec([space.wrap(str(tmpdir))], """(path):
+            def fix_path(ignored=None):
+                import _cffi_backend     # force it to be initialized
+                import sys
+                if path not in sys.path:
+                    sys.path.insert(0, path)
+            return fix_path
         """)
 
     def teardown_method(self, meth):
@@ -86,17 +89,20 @@
 
 
     def test_constant_1(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
@@ -104,6 +110,7 @@
 
     def test_dlclose(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
@@ -115,17 +122,20 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
@@ -134,12 +144,14 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
+        self.fix_path()
         from re_py_subsrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
@@ -153,6 +165,7 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
@@ -163,24 +176,28 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == "hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')

From pypy.commits at gmail.com  Fri Aug 19 03:43:00 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 00:43:00 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Import qcgc codebase
Message-ID: <57b6b884.87941c0a.620ef.0a8f@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86305:45d8ca00efff
Date: 2016-08-19 09:41 +0200
http://bitbucket.org/pypy/pypy/changeset/45d8ca00efff/

Log:	Import qcgc codebase

diff --git a/rpython/translator/c/src/qcgc/allocator.c b/rpython/translator/c/src/qcgc/allocator.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/allocator.c
@@ -0,0 +1,293 @@
+#include "allocator.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <string.h>
+
+QCGC_STATIC size_t bytes_to_cells(size_t bytes);
+
+QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells);
+QCGC_STATIC cell_t *bump_allocator_allocate(size_t cells);
+QCGC_STATIC void bump_allocator_advance(size_t cells);
+
+QCGC_STATIC bool is_small(size_t cells);
+QCGC_STATIC size_t small_index(size_t cells);
+QCGC_STATIC size_t large_index(size_t cells);
+QCGC_STATIC size_t small_index_to_cells(size_t index);
+
+QCGC_STATIC cell_t *fit_allocator_allocate(size_t cells);
+QCGC_STATIC cell_t *fit_allocator_small_first_fit(size_t index, size_t cells);
+QCGC_STATIC cell_t *fit_allocator_large_fit(size_t index, size_t cells);
+QCGC_STATIC cell_t *fit_allocator_large_first_fit(size_t index, size_t cells);
+
+QCGC_STATIC bool valid_block(cell_t *ptr, size_t cells);
+
+void qcgc_allocator_initialize(void) {
+	qcgc_allocator_state.arenas =
+		qcgc_arena_bag_create(QCGC_ARENA_BAG_INIT_SIZE);
+
+	// Bump Allocator
+	qcgc_allocator_state.bump_state.bump_ptr = NULL;
+	qcgc_allocator_state.bump_state.remaining_cells = 0;
+
+	// Fit Allocator
+	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
+		qcgc_allocator_state.fit_state.small_free_list[i] =
+			qcgc_linear_free_list_create(QCGC_SMALL_FREE_LIST_INIT_SIZE);
+	}
+
+	for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
+		qcgc_allocator_state.fit_state.large_free_list[i] =
+			qcgc_exp_free_list_create(QCGC_LARGE_FREE_LIST_INIT_SIZE);
+	}
+}
+
+void qcgc_allocator_destroy(void) {
+	// Fit Allocator
+	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
+		free(qcgc_allocator_state.fit_state.small_free_list[i]);
+	}
+
+	for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
+		free(qcgc_allocator_state.fit_state.large_free_list[i]);
+	}
+
+	// Arenas
+	size_t arena_count = qcgc_allocator_state.arenas->count;
+	for (size_t i = 0; i < arena_count; i++) {
+		qcgc_arena_destroy(qcgc_allocator_state.arenas->items[i]);
+	}
+
+	free(qcgc_allocator_state.arenas);
+}
+
+cell_t *qcgc_allocator_allocate(size_t bytes) {
+	size_t size_in_cells = bytes_to_cells(bytes);
+#if CHECKED
+	assert(size_in_cells > 0);
+	assert(size_in_cells <= QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
+#endif
+	cell_t *result;
+
+	// TODO: Implement switch for bump/fit allocator
+	if (true) {
+		result = bump_allocator_allocate(size_in_cells);
+	} else {
+		result = fit_allocator_allocate(size_in_cells);
+	}
+
+	qcgc_arena_mark_allocated(result, size_in_cells);
+#if QCGC_INIT_ZERO
+	memset(result, 0, bytes);
+#endif
+	return result;
+}
+
+void qcgc_fit_allocator_add(cell_t *ptr, size_t cells) {
+	if (cells > 0) {
+		if (is_small(cells)) {
+			size_t index = small_index(cells);
+			qcgc_allocator_state.fit_state.small_free_list[index] =
+				qcgc_linear_free_list_add(
+						qcgc_allocator_state.fit_state.small_free_list[index],
+						ptr);
+		} else {
+			size_t index = large_index(cells);
+			qcgc_allocator_state.fit_state.large_free_list[index] =
+				qcgc_exp_free_list_add(
+						qcgc_allocator_state.fit_state.large_free_list[index],
+						(struct exp_free_list_item_s) {ptr, cells});
+		}
+	}
+}
+
+QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells) {
+	qcgc_allocator_state.bump_state.bump_ptr = ptr;
+	qcgc_allocator_state.bump_state.remaining_cells = cells;
+}
+
+QCGC_STATIC void bump_allocator_advance(size_t cells) {
+	qcgc_allocator_state.bump_state.bump_ptr += cells;
+	qcgc_allocator_state.bump_state.remaining_cells -= cells;
+}
+
+QCGC_STATIC cell_t *bump_allocator_allocate(size_t cells) {
+	if (cells > qcgc_allocator_state.bump_state.remaining_cells) {
+		// Grab a new arena
+		arena_t *arena = qcgc_arena_create();
+		bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
+				QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
+		qcgc_allocator_state.arenas =
+			qcgc_arena_bag_add(qcgc_allocator_state.arenas, arena);
+	}
+	cell_t *result = qcgc_allocator_state.bump_state.bump_ptr;
+	bump_allocator_advance(cells);
+	return result;
+}
+
+QCGC_STATIC cell_t *fit_allocator_allocate(size_t cells) {
+	cell_t *result;
+
+	if (is_small(cells)) {
+		size_t index = small_index(cells);
+		result = fit_allocator_small_first_fit(index, cells);
+	} else {
+		size_t index = large_index(cells);
+		result = fit_allocator_large_fit(index, cells);
+	}
+
+	if (result == NULL) {
+		// No valid block found
+		result = bump_allocator_allocate(cells);
+	}
+
+	return result;
+}
+
+QCGC_STATIC cell_t *fit_allocator_small_first_fit(size_t index, size_t cells) {
+#if CHECKED
+	assert(small_index_to_cells(index) >= cells);
+#endif
+	cell_t *result = NULL;
+	for ( ; index < QCGC_SMALL_FREE_LISTS; index++) {
+		linear_free_list_t *free_list =
+			qcgc_allocator_state.fit_state.small_free_list[index];
+		size_t list_cell_size = small_index_to_cells(index);
+
+		while (free_list->count > 0) {
+			result = free_list->items[free_list->count - 1];
+			free_list = qcgc_linear_free_list_remove_index(free_list,
+					free_list->count - 1);
+
+			// Check whether block is still valid
+			if (valid_block(result, list_cell_size)) {
+				qcgc_fit_allocator_add(result + cells, list_cell_size - cells);
+				break;
+			} else {
+				result = NULL;
+			}
+		}
+
+		qcgc_allocator_state.fit_state.small_free_list[index] = free_list;
+		if (result != NULL) {
+			return result;
+		}
+	}
+	return fit_allocator_large_first_fit(0, cells);
+}
+
+QCGC_STATIC cell_t *fit_allocator_large_fit(size_t index, size_t cells) {
+#if CHECKED
+	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP) <= cells);
+	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP + 1) > cells);
+#endif
+	exp_free_list_t *free_list =
+		qcgc_allocator_state.fit_state.large_free_list[index];
+	size_t best_fit_index = free_list->count;
+
+	cell_t *result = NULL;
+	size_t best_fit_cells = SIZE_MAX;
+	size_t i = 0;
+	while (i < free_list->count) {
+		if (valid_block(free_list->items[i].ptr, free_list->items[i].size)) {
+			if (free_list->items[i].size >= cells &&
+					free_list->items[i].size < best_fit_cells) {
+				result = free_list->items[i].ptr;
+				best_fit_cells = free_list->items[i].size;
+				best_fit_index = i;
+			}
+			i++;
+		} else {
+			free_list = qcgc_exp_free_list_remove_index(free_list, i);
+			// NO i++ !
+		}
+
+		if (best_fit_cells == cells) {
+			break;
+		}
+	}
+
+	if (result != NULL) {
+		// Best fit was found
+		assert(best_fit_index < free_list->count);
+		free_list = qcgc_exp_free_list_remove_index(free_list, best_fit_index);
+		qcgc_fit_allocator_add(result + cells, best_fit_cells - cells);
+	} else {
+		// No best fit, go for first fit
+		result = fit_allocator_large_first_fit(index + 1, cells);
+	}
+	qcgc_allocator_state.fit_state.large_free_list[index] = free_list;
+	return result;
+}
+
+QCGC_STATIC cell_t *fit_allocator_large_first_fit(size_t index, size_t cells) {
+#if CHECKED
+	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP) >= cells);
+#endif
+	cell_t *result = NULL;
+	for ( ; index < QCGC_LARGE_FREE_LISTS; index++) {
+		exp_free_list_t *free_list =
+			qcgc_allocator_state.fit_state.large_free_list[index];
+		while(free_list->count > 0) {
+			struct exp_free_list_item_s item =
+				free_list->items[free_list->count - 1];
+			free_list = qcgc_exp_free_list_remove_index(free_list,
+					free_list->count - 1);
+
+			// Check whether block is still valid
+			if (valid_block(item.ptr, item.size)) {
+				qcgc_fit_allocator_add(item.ptr + cells, item.size - cells);
+				result = item.ptr;
+				break;
+			}
+		}
+		qcgc_allocator_state.fit_state.large_free_list[index] = free_list;
+		if (result != NULL) {
+			return result;
+		}
+	}
+	return NULL;
+}
+
+QCGC_STATIC size_t bytes_to_cells(size_t bytes) {
+	return (bytes + sizeof(cell_t) - 1) / sizeof(cell_t);
+}
+
+QCGC_STATIC bool is_small(size_t cells) {
+	return cells <= QCGC_SMALL_FREE_LISTS;
+}
+
+QCGC_STATIC size_t small_index(size_t cells) {
+#if CHECKED
+	assert(is_small(cells));
+#endif
+	return cells - 1;
+}
+
+QCGC_STATIC size_t large_index(size_t cells) {
+#if CHECKED
+	assert(!is_small(cells));
+#endif
+	// shift such that the meaningless part disappears, i.e. everything that
+	// belongs into the first free list will become 1.
+	cells = cells >> QCGC_LARGE_FREE_LIST_FIRST_EXP;
+
+	// calculates floor(log(cells))
+	return (8 * sizeof(unsigned long)) - __builtin_clzl(cells) - 1;
+}
+
+QCGC_STATIC size_t small_index_to_cells(size_t index) {
+#if CHECKED
+	assert(index < QCGC_SMALL_FREE_LISTS);
+#endif
+	return index + 1;
+}
+
+QCGC_STATIC bool valid_block(cell_t *ptr, size_t cells) {
+#if CHECKED
+	assert(ptr != NULL);
+	assert(cells > 0);
+#endif
+	return (qcgc_arena_get_blocktype(ptr) == BLOCK_FREE &&
+			qcgc_arena_get_blocktype(ptr + cells) != BLOCK_EXTENT);
+}
diff --git a/rpython/translator/c/src/qcgc/allocator.h b/rpython/translator/c/src/qcgc/allocator.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/allocator.h
@@ -0,0 +1,75 @@
+#pragma once
+
+#include "config.h"
+
+#include <stddef.h>
+
+#include "arena.h"
+#include "bag.h"
+#include "object.h"
+
+/**
+ * Free lists:
+ *
+ * Small free lists:
+ *                +---+---+-----+----+
+ * index:         | 0 | 1 | ... | 30 |
+ *                +---+---+-----+----+
+ * size (cells):  | 1 | 2 | ... | 31 |
+ *                +---+---+-----+----+
+ *
+ * Large free lists:
+ *                        +-----+-----+-----+---------+
+ * index:                 |  0  |  1  | ... |    x    |
+ *                        +-----+-----+-----+---------+
+ * minimal size (cells):  | 2^5 | 2^6 | ... | 2^(x+5) |
+ *                        +-----+-----+-----+---------+
+ *
+ * where x is chosen such that x + 5 + 1 = QCGC_ARENA_SIZE_EXP - 4 (i.e. the
+ * next bin would hold chunks that have the size of at least one arena size,
+ * which is impossible as an arena contains overhead)
+ */
+
+#define QCGC_LARGE_FREE_LISTS (QCGC_ARENA_SIZE_EXP - 4 - QCGC_LARGE_FREE_LIST_FIRST_EXP)
+
+#define QCGC_SMALL_FREE_LISTS ((1<<QCGC_LARGE_FREE_LIST_FIRST_EXP) - 1)
+
+struct qcgc_allocator_state {
+	arena_bag_t *arenas;
+	struct bump_state {
+		cell_t *bump_ptr;
+		size_t remaining_cells;
+	} bump_state;
+	struct fit_state {
+		linear_free_list_t *small_free_list[QCGC_SMALL_FREE_LISTS];
+		exp_free_list_t *large_free_list[QCGC_LARGE_FREE_LISTS];
+	} fit_state;
+} qcgc_allocator_state;
+
+/**
+ * Initialize allocator
+ */
+void qcgc_allocator_initialize(void);
+
+/**
+ * Destroy allocator
+ */
+void qcgc_allocator_destroy(void);
+
+/**
+ * Allocate new memory region
+ *
+ * @param	bytes	Desired size of the memory region in bytes
+ * @return	Pointer to memory large enough to hold size bytes, NULL in case of
+ *			errors, already zero initialized if QCGC_INIT_ZERO is set
+ */
+cell_t *qcgc_allocator_allocate(size_t bytes);
+
+
+/**
+ * Add memory to free lists
+ *
+ * @param	ptr		Pointer to memory region
+ * @param	cells	Size of memory region in cells
+ */
+void qcgc_fit_allocator_add(cell_t *ptr, size_t cells);
diff --git a/rpython/translator/c/src/qcgc/arena.c b/rpython/translator/c/src/qcgc/arena.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/arena.c
@@ -0,0 +1,322 @@
+#include "arena.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "allocator.h"
+#include "event_logger.h"
+
+/**
+ * Internal functions
+ */
+QCGC_STATIC blocktype_t get_blocktype(arena_t *arena, size_t index);
+QCGC_STATIC void set_blocktype(arena_t *arena, size_t index, blocktype_t type);
+
+arena_t *qcgc_arena_create(void) {
+	qcgc_event_logger_log(EVENT_NEW_ARENA, 0, NULL);
+
+	arena_t *result;
+	// Linux: MAP_ANONYMOUS is initialized to zero
+	cell_t *mem = (cell_t *) mmap(0, 2 * QCGC_ARENA_SIZE,
+			PROT_READ | PROT_WRITE,
+			MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+	if (mem == MAP_FAILED) {
+		// ERROR: OUT OF MEMORY
+		return NULL;
+	}
+	if (mem != qcgc_arena_addr(mem)->cells) {
+		// Not aligned -> align
+		cell_t *aligned_mem = (cell_t *)(
+				(intptr_t) qcgc_arena_addr(mem) + QCGC_ARENA_SIZE);
+		size_t size_before = (size_t)((intptr_t) aligned_mem - (intptr_t) mem);
+		size_t size_after = QCGC_ARENA_SIZE - size_before;
+
+		munmap((void *) mem, size_before);
+		munmap((void *)((intptr_t) aligned_mem + QCGC_ARENA_SIZE), size_after);
+		result = (arena_t *) aligned_mem;
+	} else {
+		// free second half
+		munmap((void *)((intptr_t) mem + QCGC_ARENA_SIZE), QCGC_ARENA_SIZE);
+		result = (arena_t *) mem;
+	}
+
+	// Init bitmaps: One large free block
+	qcgc_arena_set_bitmap_entry(result->mark_bitmap, QCGC_ARENA_FIRST_CELL_INDEX, true);
+
+	// Create gray stack
+	result->gray_stack = qcgc_gray_stack_create(QCGC_GRAY_STACK_INIT_SIZE);
+	return result;
+}
+
+void qcgc_arena_destroy(arena_t *arena) {
+#if CHECKED
+	assert(arena != NULL);
+#endif
+	free(arena->gray_stack);
+	munmap((void *) arena, QCGC_ARENA_SIZE);
+}
+
+arena_t *qcgc_arena_addr(cell_t *ptr) {
+	return (arena_t *)((intptr_t) ptr & ~(QCGC_ARENA_SIZE - 1));
+}
+
+size_t qcgc_arena_cell_index(cell_t *ptr) {
+	return (size_t)((intptr_t) ptr & (QCGC_ARENA_SIZE - 1)) >> 4;
+}
+
+bool qcgc_arena_get_bitmap_entry(uint8_t *bitmap, size_t index) {
+#if CHECKED
+	assert(bitmap != NULL);
+#endif
+	return (((bitmap[index / 8] >> (index % 8)) & 0x1) == 0x01);
+}
+
+void qcgc_arena_set_bitmap_entry(uint8_t *bitmap, size_t index, bool value) {
+#if CHECKED
+	assert(bitmap != NULL);
+#endif
+	if (value) {
+		bitmap[index / 8] |= 1<<(index % 8);
+	} else {
+		bitmap[index / 8] &= ~(1<<(index % 8));
+	}
+}
+
+QCGC_STATIC blocktype_t get_blocktype(arena_t *arena, size_t index) {
+#if CHECKED
+	assert(arena != NULL);
+#endif
+	uint8_t block_bit = qcgc_arena_get_bitmap_entry(arena->block_bitmap, index);
+	uint8_t mark_bit = qcgc_arena_get_bitmap_entry(arena->mark_bitmap, index);
+
+	if (block_bit) {
+		if (mark_bit) {
+			return BLOCK_BLACK;
+		} else {
+			return BLOCK_WHITE;
+		}
+	} else {
+		if (mark_bit) {
+			return BLOCK_FREE;
+		} else {
+			return BLOCK_EXTENT;
+		}
+	}
+}
+
+blocktype_t qcgc_arena_get_blocktype(cell_t *ptr) {
+	size_t index = qcgc_arena_cell_index(ptr);
+	arena_t *arena = qcgc_arena_addr(ptr);
+
+	return get_blocktype(arena, index);
+}
+
+QCGC_STATIC void set_blocktype(arena_t *arena, size_t index, blocktype_t type) {
+#if CHECKED
+	assert(arena != NULL);
+#endif
+	switch(type) {
+		case BLOCK_EXTENT:
+			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, false);
+			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, false);
+			break;
+		case BLOCK_FREE:
+			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, false);
+			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, true);
+			break;
+		case BLOCK_WHITE:
+			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, true);
+			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, false);
+			break;
+		case BLOCK_BLACK:
+			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, true);
+			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, true);
+			break;
+	}
+}
+
+void qcgc_arena_set_blocktype(cell_t *ptr, blocktype_t type) {
+	size_t index = qcgc_arena_cell_index(ptr);
+	arena_t *arena = qcgc_arena_addr(ptr);
+	set_blocktype(arena, index, type);
+}
+
+void qcgc_arena_mark_allocated(cell_t *ptr, size_t cells) {
+	size_t index = qcgc_arena_cell_index(ptr);
+	arena_t *arena = qcgc_arena_addr(ptr);
+	set_blocktype(arena, index, BLOCK_WHITE);
+	size_t index_of_next_block = index + cells;
+	if (index_of_next_block < QCGC_ARENA_CELLS_COUNT &&
+			get_blocktype(arena, index_of_next_block) == BLOCK_EXTENT) {
+		set_blocktype(arena, index_of_next_block, BLOCK_FREE);
+	}
+}
+
+void qcgc_arena_mark_free(cell_t *ptr) {
+	qcgc_arena_set_blocktype(ptr, BLOCK_FREE);
+	// No coalescing, collector will do this
+}
+
+bool qcgc_arena_sweep(arena_t *arena) {
+#if CHECKED
+	assert(arena != NULL);
+#endif
+	bool free = true;
+	bool coalesce = false;
+	size_t last_free_cell = QCGC_ARENA_FIRST_CELL_INDEX;
+	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
+			cell < QCGC_ARENA_CELLS_COUNT;
+			cell++) {
+		switch (qcgc_arena_get_blocktype(arena->cells + cell)) {
+			case BLOCK_EXTENT:
+				break;
+			case BLOCK_FREE:
+				if (coalesce) {
+					set_blocktype(arena, cell, BLOCK_EXTENT);
+				} else {
+					last_free_cell = cell;
+				}
+				coalesce = true;
+				break;
+			case BLOCK_WHITE:
+				if (coalesce) {
+					set_blocktype(arena, cell, BLOCK_EXTENT);
+				} else {
+					set_blocktype(arena, cell, BLOCK_FREE);
+					last_free_cell = cell;
+				}
+				coalesce = true;
+				break;
+			case BLOCK_BLACK:
+				set_blocktype(arena, cell, BLOCK_WHITE);
+				if (coalesce) {
+					qcgc_fit_allocator_add(&(arena->cells[last_free_cell]),
+							cell - last_free_cell);
+				}
+				free = false;
+				coalesce = false;
+				break;
+		}
+	}
+	if (coalesce && !free) {
+		qcgc_fit_allocator_add(&(arena->cells[last_free_cell]),
+							QCGC_ARENA_CELLS_COUNT - last_free_cell);
+	}
+	return free;
+}
+
+bool qcgc_arena_is_empty(arena_t *arena) {
+#if CHECKED
+	assert(arena != NULL);
+#endif
+	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
+			cell < QCGC_ARENA_CELLS_COUNT;
+			cell++) {
+		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+			case BLOCK_WHITE: // Fall through
+			case BLOCK_BLACK:
+				return false;
+
+			default:
+				break;
+		}
+	}
+	return true;
+}
+
+bool qcgc_arena_is_coalesced(arena_t *arena) {
+#if CHECKED
+	assert(arena != NULL);
+#endif
+	bool prev_was_free = false;
+	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
+			cell < QCGC_ARENA_CELLS_COUNT;
+			cell++) {
+		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+			case BLOCK_WHITE: // Fall through
+			case BLOCK_BLACK:
+				prev_was_free = false;
+				break;
+
+			case BLOCK_FREE:
+				if (prev_was_free) {
+					return false;
+				} else {
+					prev_was_free = true;
+				}
+				break;
+
+			case BLOCK_EXTENT:
+				break;
+		}
+	}
+	return true;
+}
+
+size_t qcgc_arena_free_blocks(arena_t *arena) {
+#if CHECKED
+	assert(arena != NULL);
+#endif
+	size_t result = 0;
+	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
+			cell < QCGC_ARENA_CELLS_COUNT;
+			cell++) {
+		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+			case BLOCK_WHITE: // Fall through
+			case BLOCK_BLACK:
+			case BLOCK_EXTENT:
+				break;
+
+			case BLOCK_FREE:
+				result++;
+				break;
+		}
+	}
+	return result;
+}
+
+size_t qcgc_arena_white_blocks(arena_t *arena) {
+#if CHECKED
+	assert(arena != NULL);
+#endif
+	size_t result = 0;
+	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
+			cell < QCGC_ARENA_CELLS_COUNT;
+			cell++) {
+		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+			case BLOCK_BLACK:	// Fall through
+			case BLOCK_EXTENT:
+			case BLOCK_FREE:
+				break;
+
+			case BLOCK_WHITE:
+				result++;
+				break;
+		}
+	}
+	return result;
+}
+
+size_t qcgc_arena_black_blocks(arena_t *arena) {
+#if CHECKED
+	assert(arena != NULL);
+#endif
+	size_t result = 0;
+	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
+			cell < QCGC_ARENA_CELLS_COUNT;
+			cell++) {
+		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+			case BLOCK_WHITE: // Fall through
+			case BLOCK_FREE:
+			case BLOCK_EXTENT:
+				break;
+
+			case BLOCK_BLACK:
+				result++;
+				break;
+		}
+	}
+	return result;
+}
diff --git a/rpython/translator/c/src/qcgc/arena.h b/rpython/translator/c/src/qcgc/arena.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/arena.h
@@ -0,0 +1,188 @@
+/**
+ * @file	arena.h
+ */
+
+#pragma once
+
+#include "config.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/types.h>
+
+#include "gray_stack.h"
+
+#define QCGC_ARENA_SIZE (1<<QCGC_ARENA_SIZE_EXP)
+
+#define QCGC_ARENA_BITMAP_SIZE (1<<(QCGC_ARENA_SIZE_EXP - 7)) // 1 / 128
+#define QCGC_ARENA_CELLS_COUNT (1<<(QCGC_ARENA_SIZE_EXP - 4))
+
+#define QCGC_ARENA_FIRST_CELL_INDEX (1<<(QCGC_ARENA_SIZE_EXP - 10))
+
+/**
+ * @typedef cell_t
+ * The smallest unit of memory that can be addressed and allocated in arenas.
+ */
+typedef uint8_t cell_t[16];
+
+/**
+ * @typedef arena_t
+ * Arena object
+ */
+typedef union {
+	struct {
+		union {
+			gray_stack_t *gray_stack;
+			uint8_t block_bitmap[QCGC_ARENA_BITMAP_SIZE];
+		};
+		uint8_t mark_bitmap[QCGC_ARENA_BITMAP_SIZE];
+	};
+	cell_t cells[QCGC_ARENA_CELLS_COUNT];
+} arena_t;
+
+/**
+ * @typedef blocktype_t
+ * Blocktypes:
+ * - BLOCK_EXTENT	Extension of previous block
+ * - BLOCK_FREE		Free block
+ * - BLOCK_WHITE	Allocated block, marked white
+ * - BLOCK_BLACK	Allocated block, marked black
+ */
+typedef enum blocktype {
+	BLOCK_EXTENT,
+	BLOCK_FREE,
+	BLOCK_WHITE,
+	BLOCK_BLACK,
+} blocktype_t;
+
+/**
+ * Create a new arena.
+ *
+ * @return Pointer to new arena, NULL in case of errors
+ */
+arena_t *qcgc_arena_create(void);
+
+/**
+ * Destroys an arena (return to OS).
+ *
+ * @param	arena	The arena to destroy
+ */
+void qcgc_arena_destroy(arena_t *arena);
+
+/**
+ * Arena pointer for given cell.
+ *
+ * @param	ptr		Pointer to cell for which you want to know the corresponding
+ *					arena
+ * @return	The arena the pointer belongs to
+ */
+arena_t *qcgc_arena_addr(cell_t *ptr);
+
+/**
+ * Index of cell in arena.
+ *
+ * @param	ptr		Pointer to cell for which you want to know the cell index
+ * @return	Index of the cell to which ptr points to
+ */
+size_t qcgc_arena_cell_index(cell_t *ptr);
+
+/**
+ * Get bitmap value for given bitmap and cell index.
+ *
+ * @param	bitmap	Bitmap
+ * @param	index	Index of cell
+ * @return	true if bitmap entry is set, false otherwise
+ */
+bool qcgc_arena_get_bitmap_entry(uint8_t *bitmap, size_t index);
+
+/**
+ * Set bitmap value for given bitmap and cell index.
+ *
+ * @param	bitmap	Bitmap
+ * @param	index	Index of cell
+ * @param	value	true to set entry, false to reset entry
+ */
+void qcgc_arena_set_bitmap_entry(uint8_t *bitmap, size_t index, bool value);
+
+/**
+ * Get blocktype.
+ *
+ * @param	ptr		Pointer to cell for which you want to know the blocktype
+ * @return	Blocktype
+ */
+blocktype_t qcgc_arena_get_blocktype(cell_t *ptr);
+
+/**
+ * Set blocktype.
+ *
+ * @param	ptr		Pointer to cell for which you want to set the blocktype
+ * @param	type	Blocktype that should be set
+ */
+void qcgc_arena_set_blocktype(cell_t *ptr, blocktype_t type);
+
+/**
+ * Mark ptr as allocated area with given size.
+ *
+ * @param	ptr		Pointer to first cell of area
+ * @param	cells	Size in cells
+ */
+void qcgc_arena_mark_allocated(cell_t *ptr, size_t cells);
+
+/**
+ * Mark cell ptr point to as free (no coalescing).
+ *
+ * @param	ptr		Pointer to cell that should be marked as free
+ */
+void qcgc_arena_mark_free(cell_t *ptr);
+
+/**
+ * Sweep given arena.
+ *
+ * @param	arena	Arena
+ * @return	Whether arena is empty after sweeping
+ */
+bool qcgc_arena_sweep(arena_t *arena);
+
+/*******************************************************************************
+ * Debug functions                                                             *
+ ******************************************************************************/
+
+/**
+ * Check whether arena is empty.
+ *
+ * @param	arena	Arena
+ * @return	true iff given arena is empty
+ */
+bool qcgc_arena_is_empty(arena_t *arena);
+
+/**
+ * Check whether arena is coalesced (no consecutive free blocks).
+ *
+ * @param	arena	Arena
+ * @return	true iff given arena is coalesced
+ */
+bool qcgc_arena_is_coalesced(arena_t *arena);
+
+/**
+ * Count free blocks.
+ *
+ * @param	arena	Arena
+ * @return	Number of free blocks
+ */
+size_t qcgc_arena_free_blocks(arena_t *arena);
+
+/**
+ * Count white blocks.
+ *
+ * @param	arena	Arena
+ * @return	Number of white blocks
+ */
+size_t qcgc_arena_white_blocks(arena_t *arena);
+
+/**
+ * Count black blocks.
+ *
+ * @param	arena	Arena
+ * @return	Number of black blocks
+ */
+size_t qcgc_arena_black_blocks(arena_t *arena);
diff --git a/rpython/translator/c/src/qcgc/bag.c b/rpython/translator/c/src/qcgc/bag.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/bag.c
@@ -0,0 +1,7 @@
+#include "bag.h"
+
+#include <assert.h>
+
+DEFINE_BAG(arena_bag, arena_t *);
+DEFINE_BAG(linear_free_list, cell_t *);
+DEFINE_BAG(exp_free_list, struct exp_free_list_item_s);
diff --git a/rpython/translator/c/src/qcgc/bag.h b/rpython/translator/c/src/qcgc/bag.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/bag.h
@@ -0,0 +1,82 @@
+#pragma once
+
+#include "config.h"
+
+#include <stddef.h>
+#include <stdlib.h>
+
+#include "arena.h"
+
+#define DECLARE_BAG(name, type)												\
+typedef struct name##_s {													\
+	size_t size;															\
+	size_t count;															\
+	type items[];															\
+} name##_t;																	\
+																			\
+name##_t *qcgc_##name##_create(size_t size);								\
+name##_t *qcgc_##name##_add(name##_t *self, type item);						\
+name##_t *qcgc_##name##_remove_index(name##_t *self, size_t index);
+
+#define DEFINE_BAG(name, type)												\
+QCGC_STATIC size_t name##_size(size_t size);								\
+QCGC_STATIC name##_t *name##_grow(name##_t *self);							\
+QCGC_STATIC name##_t *name##_shrink(name##_t *self);						\
+																			\
+name##_t *qcgc_##name##_create(size_t size) {								\
+	name##_t *result = (name##_t *) malloc(name##_size(size));				\
+	result->size = size;													\
+	result->count = 0;														\
+	return result;															\
+}																			\
+																			\
+name##_t *qcgc_##name##_add(name##_t *self, type item) {					\
+	if (self->count >= self->size) {										\
+		self = name##_grow(self);											\
+	}																		\
+	self->items[self->count++] = item;										\
+	return self;															\
+}																			\
+																			\
+name##_t *qcgc_##name##_remove_index(name##_t *self, size_t index) {		\
+	if (index + 1 < self->count) {											\
+		self->items[index] = self->items[self->count - 1];					\
+	}																		\
+	self->count--;															\
+																			\
+	if (self->count < self->size / 4) {										\
+		self = name##_shrink(self);											\
+	}																		\
+	return self;															\
+}																			\
+																			\
+QCGC_STATIC name##_t *name##_grow(name##_t *self) {								\
+	name##_t *new_self = (name##_t *) realloc(self,							\
+			name##_size(self->size * 2));									\
+	assert(new_self != NULL);												\
+	self = new_self;														\
+	self->size *= 2;														\
+	return self;															\
+}																			\
+																			\
+QCGC_STATIC name##_t *name##_shrink(name##_t *self) {							\
+	name##_t *new_self = (name##_t *) realloc(self,							\
+			name##_size(self->size / 2));									\
+	assert(new_self != NULL);												\
+	self = new_self;														\
+	self->size /= 2;														\
+	return self;															\
+}																			\
+																			\
+QCGC_STATIC size_t name##_size(size_t size) {									\
+	return sizeof(name##_t) + size * sizeof(type);							\
+}
+
+struct exp_free_list_item_s {
+	cell_t *ptr;
+	size_t size;
+};
+
+DECLARE_BAG(arena_bag, arena_t *);
+DECLARE_BAG(linear_free_list, cell_t *);
+DECLARE_BAG(exp_free_list, struct exp_free_list_item_s);
diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#define CHECKED 1							// Enable runtime sanity checks
+
+#define QCGC_INIT_ZERO 1					// Init new objects with zero bytes
+
+/**
+ * Event logger
+ */
+#define EVENT_LOG 1							// Enable event log
+#define LOGFILE "./qcgc_events.log"			// Default logfile
+#define LOG_ALLOCATION 0					// Enable allocation log (warning:
+											// significant performance impact)
+
+#define QCGC_SHADOWSTACK_SIZE 128			// Number of initial entries for
+											// shadow stack
+#define QCGC_ARENA_BAG_INIT_SIZE 16			// Initial size of the arena bag
+#define QCGC_ARENA_SIZE_EXP 20				// Between 16 (64kB) and 20 (1MB)
+#define QCGC_LARGE_ALLOC_THRESHOLD 1<<14
+#define QCGC_MARK_LIST_SEGMENT_SIZE 64		// TODO: Tune for performance
+#define QCGC_GRAY_STACK_INIT_SIZE 128		// TODO: Tune for performance
+#define QCGC_INC_MARK_MIN 64				// TODO: Tune for performance
+
+/**
+ * Fit allocator
+ */
+#define QCGC_LARGE_FREE_LIST_FIRST_EXP 5	// First exponent of large free list
+#define QCGC_LARGE_FREE_LIST_INIT_SIZE 4	// Initial size for large free lists
+#define QCGC_SMALL_FREE_LIST_INIT_SIZE 16	// Initial size for small free lists
+
+/**
+ * DO NOT MODIFY BELOW HERE
+ */
+
+#ifdef TESTING
+#define QCGC_STATIC
+#else
+#define QCGC_STATIC static
+#endif
diff --git a/rpython/translator/c/src/qcgc/event_logger.c b/rpython/translator/c/src/qcgc/event_logger.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/event_logger.c
@@ -0,0 +1,81 @@
+#include "event_logger.h"
+
+#include <assert.h>
+#include <time.h>
+#include <stdio.h>
+
+#if EVENT_LOG
+static struct {
+	FILE *logfile;
+} event_logger_state;
+
+#endif
+
+void qcgc_event_logger_initialize(void) {
+#if EVENT_LOG
+	event_logger_state.logfile = fopen(LOGFILE, "w");
+	qcgc_event_logger_log(EVENT_LOG_START, 0, NULL);
+
+	if (event_logger_state.logfile == NULL)  {
+		fprintf(stderr, "%s\n", "Failed to create logfile.");
+	}
+#endif
+}
+
+void qcgc_event_logger_destroy(void) {
+#if EVENT_LOG
+	qcgc_event_logger_log(EVENT_LOG_STOP, 0, NULL);
+
+	if (event_logger_state.logfile != NULL) {
+		fflush(event_logger_state.logfile);
+		fclose(event_logger_state.logfile);
+		event_logger_state.logfile = NULL;
+	}
+#endif
+}
+
+void qcgc_event_logger_log(enum event_e event, uint32_t additional_data_size,
+		uint8_t *additional_data) {
+#if EVENT_LOG
+#if CHECKED
+	assert((additional_data_size == 0) == (additional_data == NULL));
+#endif // CHECKED
+	struct {
+		uint32_t sec;
+		uint32_t nsec;
+		uint8_t event_id;
+		uint32_t additional_data_size;
+	} __attribute__ ((packed)) log_entry;
+
+	if (event_logger_state.logfile != NULL) {
+		struct timespec t;
+		clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &t);
+
+		log_entry.sec = (uint32_t) t.tv_sec;
+		log_entry.nsec = (uint32_t) t.tv_nsec;
+		log_entry.event_id = (uint8_t) event;
+		log_entry.additional_data_size = additional_data_size;
+
+		// The size and nmemb fields are flipped intentionally
+		int result = 0;
+		result = fwrite(&log_entry, sizeof(log_entry), 1,
+				event_logger_state.logfile);
+		if (result != 1) {
+			fprintf(stderr, "%s\n", "Failed to write log entry.");
+			event_logger_state.logfile = NULL;
+			return;
+		}
+		if (additional_data_size > 0) {
+			result = fwrite(additional_data, additional_data_size, 1,
+					event_logger_state.logfile);
+
+			if (result != 1) {
+				fprintf(stderr, "%s\n", "Failed to write additional data.");
+				event_logger_state.logfile = NULL;
+				return;
+			}
+		}
+	}
+
+#endif // EVENT_LOG
+}
diff --git a/rpython/translator/c/src/qcgc/event_logger.h b/rpython/translator/c/src/qcgc/event_logger.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/event_logger.h
@@ -0,0 +1,42 @@
+#pragma once
+
+#include "config.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * All events
+ */
+enum event_e {
+	EVENT_LOG_START,			// = 0
+	EVENT_LOG_STOP,
+
+	EVENT_SWEEP_START,
+	EVENT_SWEEP_DONE,
+
+	EVENT_ALLOCATE_START,
+	EVENT_ALLOCATE_DONE,		// = 5
+
+	EVENT_NEW_ARENA,
+
+	EVENT_MARK_START,
+	EVENT_INCMARK_START,
+	EVENT_MARK_DONE,
+};
+
+/**
+ * Initialize logger
+ */
+void qcgc_event_logger_initialize(void);
+
+/**
+ * Destroy logger
+ */
+void qcgc_event_logger_destroy(void);
+
+/**
+ * Log event
+ */
+void qcgc_event_logger_log(enum event_e event, uint32_t additional_data_size,
+		uint8_t *additional_data);
diff --git a/rpython/translator/c/src/qcgc/gc_state.h b/rpython/translator/c/src/qcgc/gc_state.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/gc_state.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <stddef.h>
+
+#include "shadow_stack.h"
+
+/**
+ * @typedef gc_state_t
+ * Garbage collection states.
+ * - GC_PAUSE	No gc in progress
+ * - GC_MARK	Currently marking
+ * - GC_COLLECT	Currently collecting
+ */
+typedef enum gc_phase {
+	GC_PAUSE,
+	GC_MARK,
+	GC_COLLECT,
+} gc_phase_t;
+
+/**
+ * @var qcgc_state
+ *
+ * Global state of the garbage collector
+ */
+struct qcgc_state {
+	shadow_stack_t *shadow_stack;
+	size_t gray_stack_size;
+	gc_phase_t phase;
+} qcgc_state;
diff --git a/rpython/translator/c/src/qcgc/gray_stack.c b/rpython/translator/c/src/qcgc/gray_stack.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/gray_stack.c
@@ -0,0 +1,60 @@
+#include "gray_stack.h"
+
+#include <assert.h>
+#include <stdlib.h>
+
+#include "gc_state.h"
+
+QCGC_STATIC size_t gray_stack_size(size_t size);
+QCGC_STATIC gray_stack_t *gray_stack_grow(gray_stack_t *stack);
+QCGC_STATIC gray_stack_t *gray_stack_shrink(gray_stack_t *stack);
+
+gray_stack_t *qcgc_gray_stack_create(size_t size) {
+	gray_stack_t *result = (gray_stack_t *) malloc(gray_stack_size(size));
+	result->size = size;
+	result->index = 0;
+	return result;
+}
+
+gray_stack_t *qcgc_gray_stack_push(gray_stack_t *stack, object_t *item) {
+	if (stack->size == stack->index) {
+		stack = gray_stack_grow(stack);
+	}
+	stack->items[stack->index] = item;
+	stack->index++;
+	qcgc_state.gray_stack_size++;
+	return stack;
+}
+
+object_t *qcgc_gray_stack_top(gray_stack_t *stack) {
+#if CHECKED
+	assert(stack->index != 0);
+#endif
+	return stack->items[stack->index - 1];
+}
+
+gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack) {
+	// TODO: Add lower bound for size (config.h)
+	if (stack->index < stack->size / 4) {
+		stack = gray_stack_shrink(stack);
+	}
+	stack->index--;
+	qcgc_state.gray_stack_size--;
+	return stack;
+}
+
+QCGC_STATIC size_t gray_stack_size(size_t size) {
+	return (sizeof(gray_stack_t) + size * sizeof(object_t *));
+}
+
+QCGC_STATIC gray_stack_t *gray_stack_grow(gray_stack_t *stack) {
+	stack = (gray_stack_t *) realloc(stack, gray_stack_size(stack->size * 2));
+	stack->size *= 2;
+	return stack;
+}
+
+QCGC_STATIC gray_stack_t *gray_stack_shrink(gray_stack_t *stack) {
+	stack = (gray_stack_t *) realloc(stack, gray_stack_size(stack->size / 2));
+	stack->size /= 2;
+	return stack;
+}
diff --git a/rpython/translator/c/src/qcgc/gray_stack.h b/rpython/translator/c/src/qcgc/gray_stack.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/gray_stack.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "config.h"
+
+#include <stddef.h>
+
+#include "object.h"
+
+typedef struct gray_stack_s {
+	size_t index;
+	size_t size;
+	object_t *items[];
+} gray_stack_t;
+
+gray_stack_t *qcgc_gray_stack_create(size_t size);
+
+gray_stack_t *qcgc_gray_stack_push(gray_stack_t *stack, object_t *item);
+object_t *qcgc_gray_stack_top(gray_stack_t *stack);
+gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack);
diff --git a/rpython/translator/c/src/qcgc/mark_list.c b/rpython/translator/c/src/qcgc/mark_list.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/mark_list.c
@@ -0,0 +1,180 @@
+#include "mark_list.h"
+
+#include <assert.h>
+
+#include <stdlib.h>
+#include <string.h>
+
+QCGC_STATIC mark_list_t *qcgc_mark_list_grow(mark_list_t *list);
+QCGC_STATIC void qcgc_mark_list_check_invariant(mark_list_t *list);
+
+mark_list_t *qcgc_mark_list_create(size_t initial_size) {
+	size_t length = (initial_size + QCGC_MARK_LIST_SEGMENT_SIZE - 1) / QCGC_MARK_LIST_SEGMENT_SIZE;
+	length += (size_t) length == 0;
+	mark_list_t *result = (mark_list_t *)
+		malloc(sizeof(mark_list_t) + length * sizeof(object_t **));
+	result->head = 0;
+	result->tail = 0;
+	result->length = length;
+	result->insert_index = 0;
+	result->count = 0;
+	result->segments[result->head] = (object_t **)
+		calloc(QCGC_MARK_LIST_SEGMENT_SIZE, sizeof(object_t *));
+#if CHECKED
+	qcgc_mark_list_check_invariant(result);
+#endif
+	return result;
+}
+
+void qcgc_mark_list_destroy(mark_list_t *list) {
+#if CHECKED
+	qcgc_mark_list_check_invariant(list);
+#endif
+
+	size_t i = list->head;
+	while (i != list->tail) {
+		free(list->segments[i]);
+		i = (i + 1) % list->length;
+	}
+	free(list->segments[list->tail]);
+	free(list);
+}
+
+mark_list_t *qcgc_mark_list_push(mark_list_t *list, object_t *object) {
+#if CHECKED
+	assert(list != NULL);
+	assert(object != NULL);
+
+	qcgc_mark_list_check_invariant(list);
+	size_t old_count = list->count;
+#endif
+	if (list->insert_index >= QCGC_MARK_LIST_SEGMENT_SIZE) {
+		if ((list->tail + 1) % list->length == list->head) {
+			list = qcgc_mark_list_grow(list);
+		}
+		list->insert_index = 0;
+		list->tail = (list->tail + 1) % list->length;
+		list->segments[list->tail] = (object_t **)
+			calloc(QCGC_MARK_LIST_SEGMENT_SIZE, sizeof(object_t *));
+	}
+	list->segments[list->tail][list->insert_index] = object;
+	list->insert_index++;
+	list->count++;
+#if CHECKED
+	assert(list->count == old_count + 1);
+	assert(list->segments[list->tail][list->insert_index - 1] == object);
+	qcgc_mark_list_check_invariant(list);
+#endif
+	return list;
+}
+
+mark_list_t *qcgc_mark_list_push_all(mark_list_t *list,
+		object_t **objects, size_t count) {
+#if CHECKED
+	assert(list != NULL);
+	assert(objects != NULL);
+
+	qcgc_mark_list_check_invariant(list);
+
+	size_t old_count = list->count;
+	for (size_t i = 0; i < count; i++) {
+		assert(objects[i] != NULL);
+	}
+#endif
+	// FIXME: Optimize or remove
+	for (size_t i = 0; i < count; i++) {
+		list = qcgc_mark_list_push(list, objects[i]);
+	}
+#if CHECKED
+	assert(list->count == old_count + count);
+	qcgc_mark_list_check_invariant(list);
+#endif
+	return list;
+}
+
+object_t **qcgc_mark_list_get_head_segment(mark_list_t *list) {
+#if CHECKED
+	assert(list != NULL);
+	assert(list->segments[list->head] != NULL);
+	qcgc_mark_list_check_invariant(list);
+#endif
+	return list->segments[list->head];
+}
+
+mark_list_t *qcgc_mark_list_drop_head_segment(mark_list_t *list) {
+#if CHECKED
+	assert(list != NULL);
+	size_t old_head = list->head;
+	size_t old_tail = list->tail;
+	qcgc_mark_list_check_invariant(list);
+#endif
+	if (list->head != list->tail) {
+		free(list->segments[list->head]);
+		list->segments[list->head] = NULL;
+		list->head = (list->head + 1) % list->length;
+		list->count -= QCGC_MARK_LIST_SEGMENT_SIZE;
+	} else {
+		memset(list->segments[list->head], 0,
+				sizeof(object_t *) * QCGC_MARK_LIST_SEGMENT_SIZE);
+		list->insert_index = 0;
+		list->count = 0;
+	}
+#if CHECKED
+	assert(old_tail == list->tail);
+	if (old_head == old_tail) {
+		assert(old_head == list->head);
+	} else {
+		assert((old_head + 1) % list->length == list->head);
+	}
+	qcgc_mark_list_check_invariant(list);
+#endif
+	return list;
+}
+
+QCGC_STATIC mark_list_t *qcgc_mark_list_grow(mark_list_t *list) {
+#if CHECKED
+	assert(list != NULL);
+	size_t old_length = list->length;
+	size_t old_tail = list->tail;
+	qcgc_mark_list_check_invariant(list);
+#endif
+	mark_list_t *new_list = (mark_list_t *) realloc(list,
+			sizeof(mark_list_t) + 2 * list->length * sizeof(object_t **));
+	if (new_list->tail < new_list->head) {
+		memcpy(new_list->segments + new_list->length,
+				new_list->segments, (new_list->tail + 1) * sizeof(object_t **));
+		new_list->tail = new_list->length + new_list->tail;
+	}
+	new_list->length = 2 * new_list->length;
+#if CHECKED
+	assert(new_list->length == 2 * old_length);
+	if (old_tail < new_list->head) {
+		assert(new_list->tail == old_tail + old_length);
+		for (size_t i = 0; i < old_tail; i++) {
+			assert(new_list->segments[i] == new_list->segments[i + old_length]);
+		}
+	} else {
+		assert(new_list->tail == old_tail);
+	}
+	qcgc_mark_list_check_invariant(new_list);
+#endif
+	return new_list;
+}
+
+QCGC_STATIC void qcgc_mark_list_check_invariant(mark_list_t *list) {
+	assert(list->head < list->length);
+	assert(list->tail < list->length);
+	assert(list->count == (list->tail - list->head + list->length) % list->length * QCGC_MARK_LIST_SEGMENT_SIZE + list->insert_index);
+	for (size_t i = 0; i < list->length; i++) {
+		if ((list->head <= i && i <= list->tail) || (list->tail < list->head &&
+				(i <= list->tail || i >= list->head))) {
+			for (size_t j = 0; j < QCGC_MARK_LIST_SEGMENT_SIZE; j++) {
+				if (i != list->tail || j < list->insert_index) {
+					assert(list->segments[i][j] != NULL);
+				} else {
+					assert(list->segments[i][j] == NULL);
+				}
+			}
+		}
+	}
+}
diff --git a/rpython/translator/c/src/qcgc/mark_list.h b/rpython/translator/c/src/qcgc/mark_list.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/mark_list.h
@@ -0,0 +1,35 @@
+/**
+ * @file	mark_list.h
+ *
+ * Object list for marking step
+ */
+
+#pragma once
+
+#include "config.h"
+
+#include <stddef.h>
+
+#include "object.h"
+
+/**
+ * Mark list - circular buffer.
+ */
+typedef struct mark_list_s {
+	size_t head;
+	size_t tail;
+	size_t length;
+	size_t count;
+	size_t insert_index;
+	object_t **segments[];
+} mark_list_t;
+
+mark_list_t *qcgc_mark_list_create(size_t initial_size);
+void qcgc_mark_list_destroy(mark_list_t *list);
+
+mark_list_t *qcgc_mark_list_push(mark_list_t *list, object_t *object);
+mark_list_t *qcgc_mark_list_push_all(mark_list_t *list,
+		object_t **objects, size_t count);
+
+object_t **qcgc_mark_list_get_head_segment(mark_list_t *list);
+mark_list_t *qcgc_mark_list_drop_head_segment(mark_list_t *list);
diff --git a/rpython/translator/c/src/qcgc/object.h b/rpython/translator/c/src/qcgc/object.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/object.h
@@ -0,0 +1,11 @@
+#pragma once
+
+
+#include "config.h"
+#include <stdint.h>
+
+#define QCGC_GRAY_FLAG 0x01
+
+typedef struct object_s {
+	uint32_t flags;
+} object_t;
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -0,0 +1,255 @@
+#include "qcgc.h"
+
+#include <assert.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "allocator.h"
+#include "event_logger.h"
+
+// TODO: Eventually move to own header?
+#define MAX(a,b) (((a)>(b))?(a):(b))
+#define MIN(a,b) (((a)<(b))?(a):(b))
+
+void qcgc_mark(void);
+void qcgc_mark_all(void);
+void qcgc_mark_incremental(void);
+void qcgc_pop_object(object_t *object);
+void qcgc_push_object(object_t *object);
+void qcgc_sweep(void);
+
+void qcgc_initialize(void) {
+	qcgc_state.shadow_stack = qcgc_shadow_stack_create(QCGC_SHADOWSTACK_SIZE);
+	qcgc_state.gray_stack_size = 0;
+	qcgc_state.phase = GC_PAUSE;
+	qcgc_allocator_initialize();
+	qcgc_event_logger_initialize();
+}
+
+void qcgc_destroy(void) {
+	qcgc_event_logger_destroy();
+	qcgc_allocator_destroy();
+	free(qcgc_state.shadow_stack);
+}
+
+/**
+ * Shadow stack
+ */
+void qcgc_shadowstack_push(object_t *object) {
+	qcgc_state.shadow_stack =
+		qcgc_shadow_stack_push(qcgc_state.shadow_stack, object);
+}
+
+object_t *qcgc_shadowstack_pop(void) {
+	object_t *result = qcgc_shadow_stack_top(qcgc_state.shadow_stack);
+	qcgc_state.shadow_stack = qcgc_shadow_stack_pop(qcgc_state.shadow_stack);
+	return result;
+}
+
+/*******************************************************************************
+ * Write barrier                                                               *
+ ******************************************************************************/
+void qcgc_write(object_t *object) {
+#if CHECKED
+	assert(object != NULL);
+#endif
+	if ((object->flags & QCGC_GRAY_FLAG) == 0) {
+		object->flags |= QCGC_GRAY_FLAG;
+		if (qcgc_state.phase != GC_PAUSE) {
+			if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK) {
+				// This was black before, push it to gray stack again
+				arena_t *arena = qcgc_arena_addr((cell_t *) object);
+				arena->gray_stack = qcgc_gray_stack_push(
+						arena->gray_stack, object);
+			}
+		}
+	}
+}
+
+/*******************************************************************************
+ * Allocation                                                                  *
+ ******************************************************************************/
+
+object_t *qcgc_allocate(size_t size) {
+#if LOG_ALLOCATION
+	qcgc_event_logger_log(EVENT_ALLOCATE_START, sizeof(size_t),
+			(uint8_t *) &size);
+#endif
+
+	object_t *result = (object_t *) qcgc_allocator_allocate(size);
+	result->flags |= QCGC_GRAY_FLAG;
+
+#if LOG_ALLOCATION
+	qcgc_event_logger_log(EVENT_ALLOCATE_DONE, sizeof(object_t *),
+			(uint8_t *) &result);
+#endif
+	return result;
+}
+
+/*******************************************************************************
+ * Collection                                                                  *
+ ******************************************************************************/
+
+mark_color_t qcgc_get_mark_color(object_t *object) {
+#if CHECKED
+	assert(object != NULL);
+#endif
+	blocktype_t blocktype = qcgc_arena_get_blocktype((cell_t *) object);
+	bool gray = (object->flags & QCGC_GRAY_FLAG) == QCGC_GRAY_FLAG;
+	if (blocktype == BLOCK_WHITE) {
+		if (gray) {
+			return MARK_COLOR_LIGHT_GRAY;
+		} else {
+			return MARK_COLOR_WHITE;
+		}
+	} else if(blocktype == BLOCK_BLACK) {
+		if (gray) {
+			return MARK_COLOR_DARK_GRAY;
+		} else {
+			return MARK_COLOR_BLACK;
+		}
+	} else {
+#if CHECKED
+		assert(false);
+#endif
+	}
+}
+
+void qcgc_mark(void) {
+	qcgc_mark_all();
+}
+
+void qcgc_mark_all(void) {
+#if CHECKED
+	assert(qcgc_state.phase == GC_PAUSE || qcgc_state.phase == GC_MARK);
+#endif
+	qcgc_event_logger_log(EVENT_MARK_START, 0, NULL);
+
+	qcgc_state.phase = GC_MARK;
+
+	// Push all roots
+	for (size_t i = 0; i < qcgc_state.shadow_stack->count; i++) {
+		qcgc_push_object(qcgc_state.shadow_stack->items[i]);
+	}
+
+	while(qcgc_state.gray_stack_size > 0) {
+		for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
+			arena_t *arena = qcgc_allocator_state.arenas->items[i];
+			while (arena->gray_stack->index > 0) {
+				object_t *top =
+					qcgc_gray_stack_top(arena->gray_stack);
+				arena->gray_stack =
+					qcgc_gray_stack_pop(arena->gray_stack);
+				qcgc_pop_object(top);
+			}
+		}
+	}
+
+	qcgc_state.phase = GC_COLLECT;
+
+	qcgc_event_logger_log(EVENT_MARK_DONE, 0, NULL);
+}
+
+void qcgc_mark_incremental(void) {
+#if CHECKED
+	assert(qcgc_state.phase == GC_PAUSE || qcgc_state.phase == GC_MARK);
+#endif
+	unsigned long gray_stack_size = qcgc_state.gray_stack_size;
+	qcgc_event_logger_log(EVENT_INCMARK_START, sizeof(gray_stack_size),
+			(uint8_t *) &gray_stack_size);
+
+	qcgc_state.phase = GC_MARK;
+
+	// Push all roots
+	for (size_t i = 0; i < qcgc_state.shadow_stack->count; i++) {
+		qcgc_push_object(qcgc_state.shadow_stack->items[i]);
+	}
+
+	for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
+		arena_t *arena = qcgc_allocator_state.arenas->items[i];
+		size_t initial_stack_size = arena->gray_stack->index;
+		size_t to_process = MIN(arena->gray_stack->index,
+				MAX(initial_stack_size / 2, QCGC_INC_MARK_MIN));
+		while (to_process > 0) {
+			object_t *top =
+				qcgc_gray_stack_top(arena->gray_stack);
+			arena->gray_stack =
+				qcgc_gray_stack_pop(arena->gray_stack);
+			qcgc_pop_object(top);
+			to_process--;
+		}
+	}
+
+	if (qcgc_state.gray_stack_size == 0) {
+		qcgc_state.phase = GC_COLLECT;
+	}
+
+	gray_stack_size = qcgc_state.gray_stack_size;
+	qcgc_event_logger_log(EVENT_INCMARK_START, sizeof(gray_stack_size),
+			(uint8_t *) &gray_stack_size);
+}
+
+void qcgc_pop_object(object_t *object) {
+#if CHECKED
+	assert(object != NULL);
+	assert((object->flags & QCGC_GRAY_FLAG) == QCGC_GRAY_FLAG);
+	assert(qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK);
+#endif
+	object->flags &= ~QCGC_GRAY_FLAG;
+	qcgc_trace_cb(object, &qcgc_push_object);
+#if CHECKED
+	assert(qcgc_get_mark_color(object) == MARK_COLOR_BLACK);
+#endif
+}
+
+void qcgc_push_object(object_t *object) {
+#if CHECKED
+	size_t old_stack_size = qcgc_state.gray_stack_size;
+	assert(qcgc_state.phase == GC_MARK);
+#endif
+	if (object != NULL) {
+		if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_WHITE) {
+			object->flags |= QCGC_GRAY_FLAG;
+			qcgc_arena_set_blocktype((cell_t *) object, BLOCK_BLACK);
+			arena_t *arena = qcgc_arena_addr((cell_t *) object);
+			arena->gray_stack = qcgc_gray_stack_push(arena->gray_stack, object);
+		}
+	}
+#if CHECKED
+	if (object != NULL) {
+		if (old_stack_size == qcgc_state.gray_stack_size) {
+			assert(qcgc_get_mark_color(object) == MARK_COLOR_BLACK ||
+					qcgc_get_mark_color(object) == MARK_COLOR_DARK_GRAY);
+		} else {
+			assert(qcgc_state.gray_stack_size == old_stack_size + 1);
+			assert(qcgc_get_mark_color(object) == MARK_COLOR_DARK_GRAY);
+		}
+	} else {
+		assert(old_stack_size == qcgc_state.gray_stack_size);
+	}
+#endif
+}
+
+void qcgc_sweep(void) {
+#if CHECKED
+	assert(qcgc_state.phase == GC_COLLECT);
+#endif
+	unsigned long arena_count;
+	arena_count = qcgc_allocator_state.arenas->count;
+	qcgc_event_logger_log(EVENT_SWEEP_START, sizeof(arena_count),
+			(uint8_t *) &arena_count);
+
+	for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
+		qcgc_arena_sweep(qcgc_allocator_state.arenas->items[i]);
+	}
+	qcgc_state.phase = GC_PAUSE;
+
+	qcgc_event_logger_log(EVENT_SWEEP_DONE, 0, NULL);
+}
+
+void qcgc_collect(void) {
+	qcgc_mark();
+	qcgc_sweep();
+}
diff --git a/rpython/translator/c/src/qcgc/qcgc.h b/rpython/translator/c/src/qcgc/qcgc.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/qcgc.h
@@ -0,0 +1,93 @@
+/**
+ * @file	qcgc.h
+ */
+
+#pragma once
+
+#include "config.h"
+
+#include <stdint.h>
+#include <sys/types.h>
+
+#include "arena.h"
+#include "gc_state.h"
+#include "gray_stack.h"
+#include "object.h"
+
+/**
+ * @typedef mark_color
+ * Object state during collection
+ * - MARK_COLOR_WHITE		Clean and unprocessed
+ * - MARK_COLOR_LIGHT_GRAY	Dirty and unprocessed
+ * - MARK_COLOR_DARK_GRAY	Processing
+ * - MARK_COLOR_BLACK		Processed
+ */
+typedef enum mark_color {
+	MARK_COLOR_WHITE,
+	MARK_COLOR_LIGHT_GRAY,
+	MARK_COLOR_DARK_GRAY,
+	MARK_COLOR_BLACK,
+} mark_color_t;
+
+/**
+ * Initialize the garbage collector.
+ */
+void qcgc_initialize(void);
+
+/**
+ * Destroy the garbage collector.
+ */
+void qcgc_destroy(void);
+
+/**
+ * Write barrier.
+ *
+ * @param	object	Object to write to
+ */
+void qcgc_write(object_t *object);
+
+/**
+ * Allocate new memory region
+ *
+ * @param	size	Desired size of the memory region
+ * @return	Pointer to memory large enough to hold size bytes, NULL in case of
+ *			errors
+ */
+object_t *qcgc_allocate(size_t size);
+
+/**
+ * Run garbage collection.
+ */
+void qcgc_collect(void);
+
+/**
+ * Return color of object.
+ *
+ * @returs The color of the object, according to the mark algorithm.
+ */
+mark_color_t qcgc_get_mark_color(object_t *object);
+
+/**
+ * Add object to shadow stack
+ *
+ * @param	object	The object to push
+ */
+void qcgc_shadowstack_push(object_t *object);
+
+/**
+ * Pop object from shadow stack
+ *
+ * @return	Top element of the shadowstack
+ */
+object_t *qcgc_shadowstack_pop(void);
+
+/**
+ * Tracing function.
+ *
+ * This function traces an object, i.e. calls visit on every object referenced
+ * by the given object. Has to be provided by the library user.
+ *
+ * @param	object	The object to trace
+ * @param	visit	The function to be called on the referenced objects
+ */
+extern void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object));
diff --git a/rpython/translator/c/src/qcgc/shadow_stack.c b/rpython/translator/c/src/qcgc/shadow_stack.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/shadow_stack.c
@@ -0,0 +1,56 @@
+#include "shadow_stack.h"
+
+#include <assert.h>
+#include <stdlib.h>
+
+QCGC_STATIC size_t shadow_stack_size(size_t size);
+QCGC_STATIC shadow_stack_t *shadow_stack_grow(shadow_stack_t *stack);
+QCGC_STATIC shadow_stack_t *shadow_stack_shrink(shadow_stack_t *stack);
+
+shadow_stack_t *qcgc_shadow_stack_create(size_t size) {
+	shadow_stack_t *result = (shadow_stack_t *) malloc(shadow_stack_size(size));
+	result->size = size;
+	result->count = 0;
+	return result;
+}
+
+shadow_stack_t *qcgc_shadow_stack_push(shadow_stack_t *stack, object_t *item) {
+	if (stack->size == stack->count) {
+		stack = shadow_stack_grow(stack);
+	}
+	stack->items[stack->count] = item;
+	stack->count++;
+	return stack;
+}
+
+object_t *qcgc_shadow_stack_top(shadow_stack_t *stack) {
+#if CHECKED
+	assert(stack->count != 0);
+#endif
+	return stack->items[stack->count - 1];
+}
+
+shadow_stack_t *qcgc_shadow_stack_pop(shadow_stack_t *stack) {
+	// TODO: Add lower bound for size (config.h)
+	if (stack->count < stack->size / 4) {
+		stack = shadow_stack_shrink(stack);
+	}
+	stack->count--;
+	return stack;
+}
+
+QCGC_STATIC size_t shadow_stack_size(size_t size) {
+	return (sizeof(shadow_stack_t) + size * sizeof(object_t *));
+}
+
+QCGC_STATIC shadow_stack_t *shadow_stack_grow(shadow_stack_t *stack) {
+	stack = (shadow_stack_t *) realloc(stack, shadow_stack_size(stack->size * 2));
+	stack->size *= 2;
+	return stack;
+}
+
+QCGC_STATIC shadow_stack_t *shadow_stack_shrink(shadow_stack_t *stack) {
+	stack = (shadow_stack_t *) realloc(stack, shadow_stack_size(stack->size / 2));
+	stack->size /= 2;
+	return stack;
+}
diff --git a/rpython/translator/c/src/qcgc/shadow_stack.h b/rpython/translator/c/src/qcgc/shadow_stack.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/shadow_stack.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "config.h"
+
+#include <stddef.h>
+
+#include "object.h"
+
+typedef struct shadow_stack_s {
+	size_t count;
+	size_t size;
+	object_t *items[];
+} shadow_stack_t;
+
+shadow_stack_t *qcgc_shadow_stack_create(size_t size);
+
+shadow_stack_t *qcgc_shadow_stack_push(shadow_stack_t *stack, object_t *item);
+object_t *qcgc_shadow_stack_top(shadow_stack_t *stack);
+shadow_stack_t *qcgc_shadow_stack_pop(shadow_stack_t *stack);

From pypy.commits at gmail.com  Fri Aug 19 04:17:55 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 01:17:55 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Compile qcgc (WIP)
Message-ID: <57b6c0b3.c398c20a.80f8e.fa5d@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86306:07613d93dc9d
Date: 2016-08-19 10:16 +0200
http://bitbucket.org/pypy/pypy/changeset/07613d93dc9d/

Log:	Compile qcgc (WIP)

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -879,6 +879,33 @@
         [dict(prefix='gc-', include_dir='include', library_dir=library_dir)],
         symbol='GC_init')
 
+def configure_qcgc():
+    library_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..',
+        '..', 'translator', 'c', 'src', 'qcgc'))
+
+    separate_source = """
+    #include "qcgc.h"
+
+    extern void pypy_trace_cb(void *, void (*)(void *));
+
+    void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object)) {
+        pypy_trace_cb((void *) object, (void (*)(void *)) visit);
+    }
+    """
+
+    eci = ExternalCompilationInfo(
+            include_dirs = [library_dir],
+            #includes = []
+            separate_module_sources = [separate_source],
+            separate_module_files = [os.path.join(library_dir, f) for f in
+                ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",
+                    "gray_stack.c", "shadow_stack.c"]],
+            )
+    return configure_external_library(
+            'qcgc', eci, [dict(prefix='qcgc-', include_dir='include',
+                library_dir=library_dir)],
+            symbol='qcgc_initialize')
+
 if __name__ == '__main__':
     doc = """Example:
 
diff --git a/rpython/translator/c/test/test_qcgc.py b/rpython/translator/c/test/test_qcgc.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/test/test_qcgc.py
@@ -0,0 +1,54 @@
+import weakref
+
+import py
+
+from rpython.rlib import rgc, debug
+from rpython.rlib.objectmodel import (keepalive_until_here, compute_unique_id,
+    compute_hash, current_object_addr_as_int)
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.translator.c.test.test_genc import compile
+
+
+def setup_module(mod):
+    from rpython.rtyper.tool.rffi_platform import configure_qcgc
+    from rpython.translator.platform import CompilationError
+    configure_qcgc()
+
+
+class AbstractGCTestClass(object):
+    gcpolicy = "qcgc"
+    use_threads = False
+    extra_options = {}
+
+    # deal with cleanups
+    def setup_method(self, meth):
+        self._cleanups = []
+
+    def teardown_method(self, meth):
+        while self._cleanups:
+            #print "CLEANUP"
+            self._cleanups.pop()()
+
+    def getcompiled(self, func, argstypelist=[], annotatorpolicy=None,
+                    extra_options={}):
+        return compile(func, argstypelist, gcpolicy=self.gcpolicy,
+                       thread=self.use_threads, **extra_options)
+
+
+class TestUsingBoehm(AbstractGCTestClass):
+    gcpolicy = "boehm"
+
+    def test_malloc_a_lot(self):
+        def malloc_a_lot():
+            i = 0
+            while i < 10:
+                i += 1
+                a = [1] * 10
+                j = 0
+                while j < 20:
+                    j += 1
+                    a.append(j)
+        fn = self.getcompiled(malloc_a_lot)
+        fn()

From pypy.commits at gmail.com  Fri Aug 19 04:27:46 2016
From: pypy.commits at gmail.com (Raemi)
Date: Fri, 19 Aug 2016 01:27:46 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: start with newgc test
Message-ID: <57b6c302.915c1c0a.ef9e5.178c@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: quad-color-gc
Changeset: r86307:3815dc96e685
Date: 2016-08-19 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/3815dc96e685/

Log:	start with newgc test

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -895,8 +895,8 @@
 
     eci = ExternalCompilationInfo(
             include_dirs = [library_dir],
-            #includes = []
-            separate_module_sources = [separate_source],
+            # includes = [],
+            pre_include_bits = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
                 ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",
                     "gray_stack.c", "shadow_stack.c"]],
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1272,7 +1272,16 @@
     def test_long_chain_of_instances(self):
         res = self.run("long_chain_of_instances")
         assert res == 1500
-        
+
+
+class TestQCGC(UsingFrameworkTest):
+    gcpolicy = "qcgc"
+    should_be_moving = False
+    GC_CAN_MOVE = False
+    GC_CAN_SHRINK_ARRAY = False
+    removetypeptr = True
+
+
 
 class TestSemiSpaceGC(UsingFrameworkTest, snippet.SemiSpaceGCTestDefines):
     gcpolicy = "semispace"

From pypy.commits at gmail.com  Fri Aug 19 05:03:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 19 Aug 2016 02:03:58 -0700 (PDT)
Subject: [pypy-commit] pypy redirect-assembler-jitlog: (fijal,
 plan_rich) test and implement redirect assembler
Message-ID: <57b6cb7e.03121c0a.636a9.2cd4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: redirect-assembler-jitlog
Changeset: r86308:e2ee0ab78509
Date: 2016-08-19 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/e2ee0ab78509/

Log:	(fijal, plan_rich) test and implement redirect assembler

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1064,6 +1064,8 @@
         else:
             assert mc.get_relative_pos() <= 13
         mc.copy_to_raw_memory(oldadr)
+        # log the redirection of the call_assembler_* operation
+        jl.redirect_assembler(oldlooptoken, newlooptoken, target)
 
     def dump(self, text):
         if not self.verbose:
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 2
+JITLOG_VERSION = 3
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -244,6 +244,7 @@
     ('COMMON_PREFIX',),
     ('ABORT_TRACE',),
     ('SOURCE_CODE',),
+    ('REDIRECT_ASSEMBLER',),
 ]
 
 start = 0x11
@@ -309,6 +310,17 @@
     content = ''.join(list)
     jitlog_write_marked(content, len(content))
 
+def redirect_assembler(oldtoken, newtoken, target):
+    if not jitlog_enabled():
+        return
+    descr_nmr = compute_unique_id(oldtoken)
+    new_descr_nmr = compute_unique_id(newtoken)
+    list = [MARK_REDIRECT_ASSEMBLER, encode_le_addr(descr_nmr),
+            encode_le_addr(new_descr_nmr), encode_le_addr(target)]
+    content = ''.join(list)
+    jitlog_write_marked(content, len(content))
+
+
 class JitLogger(object):
     def __init__(self, cpu=None):
         self.cpu = cpu
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -6,6 +6,15 @@
 from rpython.jit.backend.model import AbstractCPU
 from rpython.jit.metainterp.history import ConstInt, ConstPtr
 from rpython.rlib.rjitlog import rjitlog as jl
+from rpython.jit.metainterp.history import AbstractDescr
+from rpython.rlib.objectmodel import compute_unique_id
+
+class FakeCallAssemblerLoopToken(AbstractDescr):
+    def __init__(self, target):
+	self._ll_function_addr = target
+
+    def repr_of_descr(self):
+        return 'looptoken'
 
 class FakeLog(object):
     def __init__(self):
@@ -109,3 +118,31 @@
         with py.test.raises(AssertionError):
             jl.commonprefix(None,None)
 
+    def test_redirect_assembler(self, tmpdir):
+        looptoken = FakeCallAssemblerLoopToken(0x0)
+        newlooptoken = FakeCallAssemblerLoopToken(0x1234)
+        #
+        logger = jl.JitLogger()
+        file = tmpdir.join('binary_file')
+        file.ensure()
+        fd = file.open('wb')
+        jl.jitlog_init(fd.fileno())
+        logger.start_new_trace(self.make_metainterp_sd(), jd_name='jdname')
+        log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
+        op = ResOperation(rop.CALL_ASSEMBLER_I, [], descr=looptoken)
+        log_trace.write([], [op])
+        jl.redirect_assembler(looptoken, newlooptoken, 0x1234)
+        #the next line will close 'fd', instead of logger.finish()
+        fd.close()
+        binary = file.read()
+        opnum = jl.encode_le_16bit(rop.CALL_ASSEMBLER_I)
+        id_looptoken = compute_unique_id(looptoken)
+        new_id_looptoken = compute_unique_id(newlooptoken)
+        end = jl.MARK_RESOP_DESCR + opnum + jl.encode_str('i0,looptoken') + \
+              jl.encode_le_addr(id_looptoken) + jl.encode_str('') + \
+              jl.MARK_REDIRECT_ASSEMBLER + \
+              jl.encode_le_addr(id_looptoken) + \
+              jl.encode_le_addr(new_id_looptoken) + \
+              jl.encode_le_addr(newlooptoken._ll_function_addr)
+        assert binary.endswith(end)
+        

From pypy.commits at gmail.com  Fri Aug 19 05:55:23 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 02:55:23 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Add transformer skeleton (WIP)
Message-ID: <57b6d78b.12331c0a.eefe3.3a6c@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86309:dfbb52a3c9a1
Date: 2016-08-19 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/dfbb52a3c9a1/

Log:	Add transformer skeleton (WIP)

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -62,7 +62,7 @@
 
     # gc
     ChoiceOption("gc", "Garbage Collection Strategy",
-                 ["boehm", "ref", "semispace", "statistics",
+                 ["boehm", "qcgc", "ref", "semispace", "statistics",
                   "generation", "hybrid", "minimark",'incminimark', "none"],
                   "ref", requires={
                      "ref": [("translation.rweakref", False), # XXX
@@ -75,6 +75,11 @@
                      "hybrid": [("translation.gctransformer", "framework")],
                      "boehm": [("translation.continuation", False),  # breaks
                                ("translation.gctransformer", "boehm")],
+                     "qcgc": [("translation.gctransformer", "framework"),
+                              ("translation.gcrootfinder", "qcgc"),
+                              ("translation.gcremovetypeptr", True),
+                              ("translation.thread", False),
+                              ("translation.rweakref", False)],
                      "minimark": [("translation.gctransformer", "framework")],
                      "incminimark": [("translation.gctransformer", "framework")],
                      },
@@ -94,7 +99,7 @@
                default=IS_64_BITS, cmdline="--gcremovetypeptr"),
     ChoiceOption("gcrootfinder",
                  "Strategy for finding GC Roots (framework GCs only)",
-                 ["n/a", "shadowstack", "asmgcc"],
+                 ["n/a", "shadowstack", "asmgcc", "qcgc"],
                  "shadowstack",
                  cmdline="--gcrootfinder",
                  requires={
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -515,6 +515,7 @@
                "hybrid": "hybrid.HybridGC",
                "minimark" : "minimark.MiniMarkGC",
                "incminimark" : "incminimark.IncrementalMiniMarkGC",
+               "qcgc": "qcgc.QCGC",
                }
     try:
         modulename, classname = classes[config.translation.gc].split('.')
diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/qcgc.py
@@ -0,0 +1,63 @@
+from rpython.memory.gc.base import GCBase
+from rpython.rtyper.lltypesystem import rffi, lltype
+
+class QCGC(GCBase):
+    _alloc_flavor_ = "raw"
+    moving_gc = False
+    needs_write_barrier = True
+    malloc_zero_filled = True
+    prebuilt_gc_objects_are_static_roots = True # XXX: ?
+    can_usually_pin_objects = False
+    object_minimal_size = 16
+    gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
+
+    typeid_is_in_field = 'tid'
+
+    TRANSLATION_PARAMS = {}
+    HDR = lltype.Struct(
+            'PYPYHDR',
+            ('hdr', rffi.COpaque('object_t')),
+            ('tid', lltype.Unsigned),
+            ('hash', lltype.Unsigned))
+    #HDR = rffi.COpaque('object_t')
+
+    def malloc_fixedsize_clear(self, typeid16, size,
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
+        raise NotImplementedError
+        ## XXX finalizers are ignored for now
+        ##ll_assert(not needs_finalizer, 'XXX needs_finalizer')
+        ##ll_assert(not is_finalizer_light, 'XXX is_finalizer_light')
+        #ll_assert(not contains_weakptr, 'contains_weakptr: use malloc_weakref')
+        ## XXX call optimized versions, e.g. if size < GC_NURSERY_SECTION
+        #return llop.stm_allocate(llmemory.GCREF, size, typeid16)
+
+    def malloc_varsize_clear(self, typeid16, length, size, itemsize,
+                             offset_to_length):
+        raise NotImplementedError
+        ## XXX be careful about overflows, and call optimized versions
+        #totalsize = size + itemsize * length
+        #totalsize = llarena.round_up_for_allocation(totalsize)
+        #obj = llop.stm_allocate(llmemory.Address, totalsize, typeid16)
+        #(obj + offset_to_length).signed[0] = length
+        #return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+
+    def collect(self, gen=1):
+        """Do a minor (gen=0) or major (gen>0) collection."""
+        raise NotImplementedError
+        #if gen > 0:
+        #    llop.stm_major_collect(lltype.Void)
+        #else:
+        #    llop.stm_minor_collect(lltype.Void)
+
+    def writebarrier_before_copy(self, source_addr, dest_addr,
+                                 source_start, dest_start, length):
+        raise NotImplementedError
+        # Possible implementation?
+        #llop.gc_writebarrier(dest_addr)
+        #return True
+
+    def identityhash(self, gcobj):
+        raise NotImplementedError
+
diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -0,0 +1,40 @@
+from rpython.annotator import model as annmodel
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.memory.gctransform.framework import (BaseFrameworkGCTransformer, BaseRootWalker)
+
+class QcgcFrameworkGCTransformer(BaseFrameworkGCTransformer):
+    def build_root_walker(self):
+        return QcgcRootWalker(self)
+
+    def _declare_functions(self, GCClass, getfn, s_gc, s_typeid16):
+        BaseFrameworkGCTransformer._declare_functions(self, GCClass, getfn,
+                                                      s_gc, s_typeid16)
+        gc = self.gcdata.gc
+        #
+        s_gcref = annmodel.SomePtr(llmemory.GCREF)
+
+        #self.malloc_weakref_ptr = self._getfn(
+        #    GCClass.malloc_weakref.im_func,
+        #    [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True),
+        #     s_gcref], s_gcref)
+        #
+        def invokecallback(root, visit_fn):
+            visit_fn(root)
+        def pypy_trace_cb(obj, visit_fn):
+            gc.trace(obj, invokecallback, visit_fn)
+        pypy_trace_cb.c_name = "pypy_trace_cb"
+        self.autoregister_ptrs.append(
+            getfn(pypy_trace_cb, [annmodel.SomeAddress(),
+                                     annmodel.SomePtr(GCClass.VISIT_FPTR)],
+                  annmodel.s_None))
+
+    def push_roots(sef, hop, keep_current_args=False):
+        raise NotImplementedError
+
+    def pop_roots(sef, hop, livevars):
+        raise NotImplementedError
+
+
+class QcgcRootWalker(BaseRootWalker):
+    def walk_stack_roots(self, collect_stack_root):
+        raise NotImplementedError
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -455,13 +455,19 @@
     def OP_GC_STACK_BOTTOM(self, funcgen, op):
         return 'pypy_asm_stack_bottom();'
 
+class QcgcFrameworkGcPolicy(BasicFrameworkGcPolicy):
+
+    def gettransformer(self, translator):
+        from rpython.memory.gctransform import qcgcframework
+        return qcgcframework.QcgcFrameworkGCTransformer(translator)
 
 name_to_gcpolicy = {
     'boehm': BoehmGcPolicy,
     'ref': RefcountingGcPolicy,
     'none': NoneGcPolicy,
     'framework+shadowstack': ShadowStackFrameworkGcPolicy,
-    'framework+asmgcc': AsmGcRootFrameworkGcPolicy
+    'framework+asmgcc': AsmGcRootFrameworkGcPolicy,
+    'framework+qcgc': QcgcFrameworkGcPolicy
 }
 
 

From pypy.commits at gmail.com  Fri Aug 19 07:12:30 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 04:12:30 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Extend skeleton (WIP?)
Message-ID: <57b6e99e.94a51c0a.e2706.5274@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86310:c0e8549cad2f
Date: 2016-08-19 13:11 +0200
http://bitbucket.org/pypy/pypy/changeset/c0e8549cad2f/

Log:	Extend skeleton (WIP?)

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -8,7 +8,7 @@
     malloc_zero_filled = True
     prebuilt_gc_objects_are_static_roots = True # XXX: ?
     can_usually_pin_objects = False
-    object_minimal_size = 16
+    object_minimal_size = 0
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
 
     typeid_is_in_field = 'tid'
@@ -61,3 +61,8 @@
     def identityhash(self, gcobj):
         raise NotImplementedError
 
+    def register_finalizer(self, fq_index, gcobj):
+        raise NotImplementedError
+
+    def get_type_id(self, obj):
+        return self.header(obj).tid
diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -1,8 +1,12 @@
-from rpython.annotator import model as annmodel
+from rpython.rtyper.llannotation import SomePtr, SomeAddress, s_None
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.memory.gctransform.framework import (BaseFrameworkGCTransformer, BaseRootWalker)
 
+VISIT_FPTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
+
 class QcgcFrameworkGCTransformer(BaseFrameworkGCTransformer):
+    autoregister_ptrs = list()
+
     def build_root_walker(self):
         return QcgcRootWalker(self)
 
@@ -11,7 +15,7 @@
                                                       s_gc, s_typeid16)
         gc = self.gcdata.gc
         #
-        s_gcref = annmodel.SomePtr(llmemory.GCREF)
+        s_gcref = SomePtr(llmemory.GCREF)
 
         #self.malloc_weakref_ptr = self._getfn(
         #    GCClass.malloc_weakref.im_func,
@@ -24,9 +28,9 @@
             gc.trace(obj, invokecallback, visit_fn)
         pypy_trace_cb.c_name = "pypy_trace_cb"
         self.autoregister_ptrs.append(
-            getfn(pypy_trace_cb, [annmodel.SomeAddress(),
-                                     annmodel.SomePtr(GCClass.VISIT_FPTR)],
-                  annmodel.s_None))
+            getfn(pypy_trace_cb, [SomeAddress(),
+                                     SomePtr(VISIT_FPTR)],
+                  s_None))
 
     def push_roots(sef, hop, keep_current_args=False):
         raise NotImplementedError
@@ -34,7 +38,6 @@
     def pop_roots(sef, hop, livevars):
         raise NotImplementedError
 
-
 class QcgcRootWalker(BaseRootWalker):
-    def walk_stack_roots(self, collect_stack_root):
+    def walk_stack_roots(self, collect_stack_root, is_minor=False):
         raise NotImplementedError

From pypy.commits at gmail.com  Fri Aug 19 07:46:44 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 04:46:44 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Extend GC/Transformer (WIP)
Message-ID: <57b6f1a4.c2a5c20a.44b61.4256@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86311:cba20e2f61b9
Date: 2016-08-19 13:46 +0200
http://bitbucket.org/pypy/pypy/changeset/cba20e2f61b9/

Log:	Extend GC/Transformer (WIP)

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -1,5 +1,5 @@
 from rpython.memory.gc.base import GCBase
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi, lltype, llgroup, llmemory
 
 class QCGC(GCBase):
     _alloc_flavor_ = "raw"
@@ -66,3 +66,29 @@
 
     def get_type_id(self, obj):
         return self.header(obj).tid
+
+    def init_gc_object(self, addr, typeid, flags=0):
+        assert flags == 0
+        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+        hdr.tid = typeid
+
+    def init_gc_object_immortal(self, addr, typeid, flags=0): # XXX: Prebuilt Objects?
+        assert flags == 0
+        self.init_gc_object(addr, typeid, flags)
+        prebuilt_hash = lltype.identityhash_nocache(ptr)
+        assert prebuilt_hash != 0
+        #
+        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+        hdr.hash = prebuilt_hash
+        #
+        # STMGC CODE:
+        #assert flags == 0
+        #assert isinstance(typeid16, llgroup.GroupMemberOffset)
+        #ptr = self.gcheaderbuilder.object_from_header(addr.ptr)
+        #prebuilt_hash = lltype.identityhash_nocache(ptr)
+        #assert prebuilt_hash != 0     # xxx probably good enough
+        ##
+        #hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+        #hdr._obj._name = typeid16.index   # debug only
+        #hdr._obj.typeid16 = typeid16
+        #hdr._obj.prebuilt_hash = prebuilt_hash
diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -32,11 +32,16 @@
                                      SomePtr(VISIT_FPTR)],
                   s_None))
 
-    def push_roots(sef, hop, keep_current_args=False):
-        raise NotImplementedError
+    def push_roots(self, hop, keep_current_args=False):
+        livevars = self.get_livevars_for_roots(hop, keep_current_args)
+        self.num_pushs += len(livevars)
+        for var in livevars:
+            hop.genop("qcgc_push_root", [var])
+        return livevars
 
-    def pop_roots(sef, hop, livevars):
-        raise NotImplementedError
+    def pop_roots(self, hop, livevars):
+        for _ in livevars: # Does not move, so no writing back
+            hop.genop("qcgc_pop_root", [])
 
 class QcgcRootWalker(BaseRootWalker):
     def walk_stack_roots(self, collect_stack_root, is_minor=False):

From pypy.commits at gmail.com  Fri Aug 19 08:24:11 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 05:24:11 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase
Message-ID: <57b6fa6b.c19d1c0a.a7ff3.7124@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86312:9561ef30edb5
Date: 2016-08-19 14:23 +0200
http://bitbucket.org/pypy/pypy/changeset/9561ef30edb5/

Log:	Update qcgc codebase

diff --git a/rpython/translator/c/src/qcgc/gc_state.h b/rpython/translator/c/src/qcgc/gc_state.h
--- a/rpython/translator/c/src/qcgc/gc_state.h
+++ b/rpython/translator/c/src/qcgc/gc_state.h
@@ -24,6 +24,7 @@
  */
 struct qcgc_state {
 	shadow_stack_t *shadow_stack;
+	shadow_stack_t *prebuilt_objects;
 	size_t gray_stack_size;
 	gc_phase_t phase;
 } qcgc_state;
diff --git a/rpython/translator/c/src/qcgc/object.h b/rpython/translator/c/src/qcgc/object.h
--- a/rpython/translator/c/src/qcgc/object.h
+++ b/rpython/translator/c/src/qcgc/object.h
@@ -5,6 +5,7 @@
 #include <stdint.h>
 
 #define QCGC_GRAY_FLAG 0x01
+#define QCGC_PREBUILT_OBJECT 0x02
 
 typedef struct object_s {
 	uint32_t flags;
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -22,6 +22,7 @@
 
 void qcgc_initialize(void) {
 	qcgc_state.shadow_stack = qcgc_shadow_stack_create(QCGC_SHADOWSTACK_SIZE);
+	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); //XXX
 	qcgc_state.gray_stack_size = 0;
 	qcgc_state.phase = GC_PAUSE;
 	qcgc_allocator_initialize();
@@ -57,7 +58,10 @@
 #endif
 	if ((object->flags & QCGC_GRAY_FLAG) == 0) {
 		object->flags |= QCGC_GRAY_FLAG;
-		if (qcgc_state.phase != GC_PAUSE) {
+		if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
+			// Save prebuilt object into list
+			qcgc_shadow_stack_push(qcgc_state.prebuilt_objects, object);
+		} else if (qcgc_state.phase != GC_PAUSE) {
 			if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK) {
 				// This was black before, push it to gray stack again
 				arena_t *arena = qcgc_arena_addr((cell_t *) object);
@@ -134,6 +138,11 @@
 		qcgc_push_object(qcgc_state.shadow_stack->items[i]);
 	}
 
+	// Trace all prebuilt objects
+	for (size_t i = 0; i < qcgc_state.prebuilt_objects->count; i++) {
+		qcgc_trace_cb(qcgc_state.prebuilt_objects->items[i], &qcgc_push_object);
+	}
+
 	while(qcgc_state.gray_stack_size > 0) {
 		for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
 			arena_t *arena = qcgc_allocator_state.arenas->items[i];
@@ -167,6 +176,11 @@
 		qcgc_push_object(qcgc_state.shadow_stack->items[i]);
 	}
 
+	// Trace all prebuilt objects
+	for (size_t i = 0; i < qcgc_state.prebuilt_objects->count; i++) {
+		qcgc_trace_cb(qcgc_state.prebuilt_objects->items[i], &qcgc_push_object);
+	}
+
 	for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
 		arena_t *arena = qcgc_allocator_state.arenas->items[i];
 		size_t initial_stack_size = arena->gray_stack->index;
@@ -210,6 +224,9 @@
 	assert(qcgc_state.phase == GC_MARK);
 #endif
 	if (object != NULL) {
+		if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
+			return;
+		}
 		if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_WHITE) {
 			object->flags |= QCGC_GRAY_FLAG;
 			qcgc_arena_set_blocktype((cell_t *) object, BLOCK_BLACK);

From pypy.commits at gmail.com  Fri Aug 19 08:47:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 19 Aug 2016 05:47:51 -0700 (PDT)
Subject: [pypy-commit] pypy redirect-assembler-jitlog: merge default
Message-ID: <57b6fff7.c1e31c0a.272c6.ae57@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: redirect-assembler-jitlog
Changeset: r86313:5e73f119f5f7
Date: 2016-08-19 14:47 +0200
http://bitbucket.org/pypy/pypy/changeset/5e73f119f5f7/

Log:	merge default

diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -356,6 +356,11 @@
 that a C-level traceback is usually of no help at all in PyPy.
 Debugging PyPy can be annoying.
 
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
 In more details:
 
 * First, please give the exact PyPy version, and the OS.
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -387,7 +387,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -406,8 +407,11 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            assert depth >= 0
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -596,7 +596,10 @@
 
         # First extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(func.func_code)
+        sig = pycode.cpython_code_signature(func.func_code)
+        argnames = sig.argnames
+        varargname = sig.varargname
+        kwargname = sig.kwargname
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -620,7 +623,9 @@
         app_sig = SignatureBuilder(func)
 
         UnwrapSpec_Check(orig_sig).apply_over(unwrap_spec, app_sig)
-        self.sig = argnames, varargname, kwargname = app_sig.signature()
+        self.sig = app_sig.signature()
+        argnames = self.sig.argnames
+        varargname = self.sig.varargname
 
         self.minargs = len(argnames)
         if varargname:
@@ -951,7 +956,7 @@
                 defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
             UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig[0])
+            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
             if defs_w:
                 alldefs_w[-len(defs_w):] = defs_w
             code = self._code
@@ -968,7 +973,7 @@
                     assert isinstance(w_default, W_Root)
                     assert argname.startswith('w_')
                     argname = argname[2:]
-                    j = self._code.sig[0].index(argname)
+                    j = self._code.sig.argnames.index(argname)
                     assert alldefs_w[j] in (UNDEFINED, None)
                     alldefs_w[j] = w_default
             first_defined = 0
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -37,7 +37,7 @@
 
 # cpython_code_signature helper
 def cpython_code_signature(code):
-    "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
+    """Return a Signature instance."""
     argcount = code.co_argcount
     varnames = code.co_varnames
     assert argcount >= 0     # annotator hint
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -55,18 +55,3 @@
         if not isinstance(other, Signature):
             return NotImplemented
         return not self == other
-
-
-    # make it look tuply for its use in the annotator
-
-    def __len__(self):
-        return 3
-
-    def __getitem__(self, i):
-        if i == 0:
-            return self.argnames
-        if i == 1:
-            return self.varargname
-        if i == 2:
-            return self.kwargname
-        raise IndexError
\ No newline at end of file
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -46,13 +46,6 @@
         assert sig.find_argname("c") == 2
         assert sig.find_argname("d") == -1
 
-    def test_tuply(self):
-        sig = Signature(["a", "b", "c"], "d", "e")
-        x, y, z = sig
-        assert x == ["a", "b", "c"]
-        assert y == "d"
-        assert z == "e"
-
 class dummy_wrapped_dict(dict):
     def __nonzero__(self):
         raise NotImplementedError
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -203,7 +203,8 @@
     name = func.__name__
     extra = ', '.join(extraargs)
     from pypy.interpreter import pycode
-    argnames, _, _ = pycode.cpython_code_signature(func.func_code)
+    sig = pycode.cpython_code_signature(func.func_code)
+    argnames = sig.argnames
     if use_closure:
         if argnames[1] == 'space':
             args = "closure, space, obj"
diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,10 +69,13 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
+        cls.w_fix_path = space.appexec([space.wrap(str(tmpdir))], """(path):
+            def fix_path(ignored=None):
+                import _cffi_backend     # force it to be initialized
+                import sys
+                if path not in sys.path:
+                    sys.path.insert(0, path)
+            return fix_path
         """)
 
     def teardown_method(self, meth):
@@ -86,17 +89,20 @@
 
 
     def test_constant_1(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
@@ -104,6 +110,7 @@
 
     def test_dlclose(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
@@ -115,17 +122,20 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
@@ -134,12 +144,14 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
+        self.fix_path()
         from re_py_subsrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
@@ -153,6 +165,7 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
@@ -163,24 +176,28 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == "hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -10,10 +10,10 @@
 
     def setup_class(cls):
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__jitlog.1')))
-        cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
-        cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
+        cls.w_mark_header = cls.space.newbytes(jl.MARK_JITLOG_HEADER)
+        cls.w_version = cls.space.newbytes(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
-        cls.w_machine = cls.space.wrap(platform.machine())
+        cls.w_machine = cls.space.newbytes(platform.machine())
         cls.w_resops = cls.space.newdict()
         space = cls.space
         for key, value in opname.items():
@@ -48,5 +48,3 @@
                 assert opnum in self.resops
                 # the name must equal
                 assert self.resops[opnum] == opname
-
-
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,7 +11,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
-from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite,
         c_fdopen, c_fileno,
         c_fopen)# for tests
 from rpython.translator import cdir
@@ -259,14 +259,14 @@
 
         # extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(callable.func_code)
+        sig = pycode.cpython_code_signature(callable.func_code)
+        assert sig.argnames[0] == 'space'
+        self.argnames = sig.argnames[1:]
+        if gil == 'pygilstate_ensure':
+            assert self.argnames[-1] == 'previous_state'
+            del self.argnames[-1]
+        assert len(self.argnames) == len(self.argtypes)
 
-        assert argnames[0] == 'space'
-        if gil == 'pygilstate_ensure':
-            assert argnames[-1] == 'previous_state'
-            del argnames[-1]
-        self.argnames = argnames[1:]
-        assert len(self.argnames) == len(self.argtypes)
         self.gil = gil
         self.result_borrowed = result_borrowed
         self.result_is_ll = result_is_ll
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -619,6 +619,7 @@
             assert os.geteuid() == self.geteuid
 
     if hasattr(os, 'setuid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setuid_error(self):
             os = self.posix
             raises(OverflowError, os.setuid, -2)
@@ -666,6 +667,7 @@
             raises(OSError, os.getpgid, 1234567)
 
     if hasattr(os, 'setgid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setgid_error(self):
             os = self.posix
             raises(OverflowError, os.setgid, -2)
diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -45,5 +45,8 @@
 
 def test_setrlimit():
     # minimal "does not crash" test
-    x = resource.getrlimit(resource.RLIMIT_CPU)
-    resource.setrlimit(resource.RLIMIT_CPU, x)
+    x, y = resource.getrlimit(resource.RLIMIT_CPU)
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))
+    x += 0.2
+    y += 0.3
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))    # truncated to ints
diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py
--- a/pypy/tool/pytest/genreportdata.py
+++ b/pypy/tool/pytest/genreportdata.py
@@ -17,7 +17,7 @@
             resultwc = py.path.svnwc(testresultdir)
             print "updating", resultwc
             resultwc.update()
-        except KeyboardInterrupt as RuntimeError:
+        except (KeyboardInterrupt, RuntimeError):
             raise
         except Exception as e: #py.process.ExecutionFailed,e:
             print >> sys.stderr, "Warning: ",e #Subversion update failed"

From pypy.commits at gmail.com  Fri Aug 19 08:48:42 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 05:48:42 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: "Fix" initialization (WIP)
Message-ID: <57b7002a.436ec20a.e855f.5fa0@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86314:d6468116edd7
Date: 2016-08-19 14:48 +0200
http://bitbucket.org/pypy/pypy/changeset/d6468116edd7/

Log:	"Fix" initialization (WIP)

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -1,5 +1,6 @@
 from rpython.memory.gc.base import GCBase
 from rpython.rtyper.lltypesystem import rffi, lltype, llgroup, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
 
 class QCGC(GCBase):
     _alloc_flavor_ = "raw"
@@ -12,13 +13,14 @@
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
 
     typeid_is_in_field = 'tid'
+    withhash_flag_is_in_field = 'hash', 0
 
     TRANSLATION_PARAMS = {}
     HDR = lltype.Struct(
             'PYPYHDR',
             ('hdr', rffi.COpaque('object_t')),
-            ('tid', lltype.Unsigned),
-            ('hash', lltype.Unsigned))
+            ('tid', lltype.Signed),
+            ('hash', lltype.Signed))
     #HDR = rffi.COpaque('object_t')
 
     def malloc_fixedsize_clear(self, typeid16, size,
@@ -70,11 +72,12 @@
     def init_gc_object(self, addr, typeid, flags=0):
         assert flags == 0
         hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        hdr.tid = typeid
+        hdr.tid = llop.combine_ushort(lltype.Signed, typeid, 0)
 
     def init_gc_object_immortal(self, addr, typeid, flags=0): # XXX: Prebuilt Objects?
         assert flags == 0
         self.init_gc_object(addr, typeid, flags)
+        ptr = self.gcheaderbuilder.object_from_header(addr.ptr)
         prebuilt_hash = lltype.identityhash_nocache(ptr)
         assert prebuilt_hash != 0
         #

From pypy.commits at gmail.com  Fri Aug 19 09:03:11 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 06:03:11 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Make some C compilation working
 (WIP)
Message-ID: <57b7038f.94a51c0a.e2706.7a9b@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86315:025a7647e086
Date: 2016-08-19 15:02 +0200
http://bitbucket.org/pypy/pypy/changeset/025a7647e086/

Log:	Make some C compilation working (WIP)

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -18,7 +18,7 @@
     TRANSLATION_PARAMS = {}
     HDR = lltype.Struct(
             'PYPYHDR',
-            ('hdr', rffi.COpaque('object_t')),
+            ('hdr', rffi.COpaque('object_t', hints={"is_qcgc_header": True})),
             ('tid', lltype.Signed),
             ('hash', lltype.Signed))
     #HDR = rffi.COpaque('object_t')
@@ -72,7 +72,7 @@
     def init_gc_object(self, addr, typeid, flags=0):
         assert flags == 0
         hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        hdr.tid = llop.combine_ushort(lltype.Signed, typeid, 0)
+        hdr.tid = typeid.index
 
     def init_gc_object_immortal(self, addr, typeid, flags=0): # XXX: Prebuilt Objects?
         assert flags == 0
@@ -83,6 +83,7 @@
         #
         hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
         hdr.hash = prebuilt_hash
+        #hdr._obj._name = typeid.index
         #
         # STMGC CODE:
         #assert flags == 0
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -976,12 +976,33 @@
                 args.append('0')
         yield 'RPyOpaque_SETUP_%s(%s);' % (T.tag, ', '.join(args))
 
+class QcgcHeader_OpaqueNode(ContainerNode):
+    nodekind = 'qcgchdr'
+    globalcontainer = True
+    typename = 'object_t @'
+    implementationtypename = typename
+    _funccodegen_owner = None
+
+    def __init__(self, db, T, obj):
+        ContainerNode.__init__(self, db, T, obj)
+
+    def initializationexpr(self, decoration=''):
+        yield '{ QCGC_PREBUILT_OBJECT }'
+
+    def enum_dependencies(self):
+        return []
+
+    def basename(self):
+        return self.nodekind
+
 
 def opaquenode_factory(db, T, obj):
     if T == RuntimeTypeInfo:
         return db.gcpolicy.rtti_node_factory()(db, T, obj)
     if T.hints.get("render_structure", False):
         return ExtType_OpaqueNode(db, T, obj)
+    if T.hints.get("is_qcgc_header", False):
+        return QcgcHeader_OpaqueNode(db, T, obj)
     raise Exception("don't know about %r" % (T,))
 
 

From pypy.commits at gmail.com  Fri Aug 19 09:13:30 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 06:13:30 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Add qcgc_configure
Message-ID: <57b705fa.d32d1c0a.3ca21.7b0c@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86316:5e3b2d612dbc
Date: 2016-08-19 15:12 +0200
http://bitbucket.org/pypy/pypy/changeset/5e3b2d612dbc/

Log:	Add qcgc_configure

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -461,6 +461,21 @@
         from rpython.memory.gctransform import qcgcframework
         return qcgcframework.QcgcFrameworkGCTransformer(translator)
 
+    def compilation_info(self):
+        eci = BasicGcPolicy.compilation_info(self)
+
+        from rpython.rtyper.tool.rffi_platform import configure_qcgc
+        eci = eci.merge(configure_qcgc())
+
+        return eci
+
+    def gc_startup_code(self):
+        if sys.platform == 'win32':
+            pass # yield 'assert(GC_all_interior_pointers == 0);'
+        else:
+            yield 'GC_all_interior_pointers = 0;'
+        yield 'boehm_gc_startup_code();'
+
 name_to_gcpolicy = {
     'boehm': BoehmGcPolicy,
     'ref': RefcountingGcPolicy,

From pypy.commits at gmail.com  Fri Aug 19 09:20:43 2016
From: pypy.commits at gmail.com (Raemi)
Date: Fri, 19 Aug 2016 06:20:43 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: revert to separate_module_sources
Message-ID: <57b707ab.c75dc20a.59ed5.68f0@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: quad-color-gc
Changeset: r86317:4a05d83ac1b9
Date: 2016-08-19 15:20 +0200
http://bitbucket.org/pypy/pypy/changeset/4a05d83ac1b9/

Log:	revert to separate_module_sources

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -896,7 +896,7 @@
     eci = ExternalCompilationInfo(
             include_dirs = [library_dir],
             # includes = [],
-            pre_include_bits = [separate_source],  # XXX
+            separate_module_sources = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
                 ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",
                     "gray_stack.c", "shadow_stack.c"]],

From pypy.commits at gmail.com  Fri Aug 19 09:45:15 2016
From: pypy.commits at gmail.com (Raemi)
Date: Fri, 19 Aug 2016 06:45:15 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: try to emit functions defined by
 the gctransformer
Message-ID: <57b70d6b.d4e01c0a.a8cdb.8f8c@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: quad-color-gc
Changeset: r86318:31c6daaf8842
Date: 2016-08-19 15:44 +0200
http://bitbucket.org/pypy/pypy/changeset/31c6daaf8842/

Log:	try to emit functions defined by the gctransformer

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -208,6 +208,8 @@
                 self.graphs_to_inline[graph] = True
             return annhelper.graph2const(graph)
 
+        self.autoregister_ptrs = []
+
         self.frameworkgc_setup_ptr = getfn(frameworkgc_setup, [],
                                            annmodel.s_None)
         # for tests
diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -5,7 +5,6 @@
 VISIT_FPTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
 
 class QcgcFrameworkGCTransformer(BaseFrameworkGCTransformer):
-    autoregister_ptrs = list()
 
     def build_root_walker(self):
         return QcgcRootWalker(self)
@@ -28,8 +27,8 @@
             gc.trace(obj, invokecallback, visit_fn)
         pypy_trace_cb.c_name = "pypy_trace_cb"
         self.autoregister_ptrs.append(
-            getfn(pypy_trace_cb, [SomeAddress(),
-                                     SomePtr(VISIT_FPTR)],
+            getfn(pypy_trace_cb,
+                  [SomeAddress(), SomePtr(VISIT_FPTR)],
                   s_None))
 
     def push_roots(self, hop, keep_current_args=False):
diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -895,7 +895,7 @@
 
     eci = ExternalCompilationInfo(
             include_dirs = [library_dir],
-            # includes = [],
+            includes = ["qcgc.h"],
             separate_module_sources = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
                 ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -333,6 +333,8 @@
         return FrameworkGcRuntimeTypeInfo_OpaqueNode
 
     def gc_startup_code(self):
+        for c_fnptr in self.db.gctransformer.autoregister_ptrs:
+            self.db.get(c_fnptr.value)
         fnptr = self.db.gctransformer.frameworkgc_setup_ptr.value
         yield '%s();' % (self.db.get(fnptr),)
 
@@ -469,12 +471,13 @@
 
         return eci
 
-    def gc_startup_code(self):
-        if sys.platform == 'win32':
-            pass # yield 'assert(GC_all_interior_pointers == 0);'
-        else:
-            yield 'GC_all_interior_pointers = 0;'
-        yield 'boehm_gc_startup_code();'
+    # def gc_startup_code(self):
+    #     super(QcgcFrameworkGcPolicy, self).gc_startup_code()
+        # if sys.platform == 'win32':
+        #     pass # yield 'assert(GC_all_interior_pointers == 0);'
+        # else:
+        #     yield 'GC_all_interior_pointers = 0;'
+        # yield 'boehm_gc_startup_code();'
 
 name_to_gcpolicy = {
     'boehm': BoehmGcPolicy,

From pypy.commits at gmail.com  Fri Aug 19 09:52:34 2016
From: pypy.commits at gmail.com (Raemi)
Date: Fri, 19 Aug 2016 06:52:34 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: remove configure_external_library,
 as it seems to compile the GC separately, which is not possible
Message-ID: <57b70f22.a710c20a.b630c.6fe1@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: quad-color-gc
Changeset: r86319:c3000b80ab0b
Date: 2016-08-19 15:51 +0200
http://bitbucket.org/pypy/pypy/changeset/c3000b80ab0b/

Log:	remove configure_external_library, as it seems to compile the GC
	separately, which is not possible

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -901,10 +901,7 @@
                 ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",
                     "gray_stack.c", "shadow_stack.c"]],
             )
-    return configure_external_library(
-            'qcgc', eci, [dict(prefix='qcgc-', include_dir='include',
-                library_dir=library_dir)],
-            symbol='qcgc_initialize')
+    return eci
 
 if __name__ == '__main__':
     doc = """Example:

From pypy.commits at gmail.com  Fri Aug 19 10:12:25 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 19 Aug 2016 07:12:25 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Implement push and pop
Message-ID: <57b713c9.03121c0a.636a9.9d2b@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86320:bc2d69f739ed
Date: 2016-08-19 16:11 +0200
http://bitbucket.org/pypy/pypy/changeset/bc2d69f739ed/

Log:	Implement push and pop

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -943,3 +943,10 @@
                 cdecl(typename, ''),
                 self.expr(op.args[0]),
                 self.expr(op.result))
+
+    def OP_QCGC_PUSH_ROOT(self, op):
+        obj = self.expr(op.args[0])
+        return 'qcgc_shadowstack_push((object_t *) %s);' % (obj,)
+
+    def OP_QCGC_POP_ROOT(self, op):
+        return 'qcgc_shadowstack_pop();'
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -471,13 +471,11 @@
 
         return eci
 
-    # def gc_startup_code(self):
-    #     super(QcgcFrameworkGcPolicy, self).gc_startup_code()
-        # if sys.platform == 'win32':
-        #     pass # yield 'assert(GC_all_interior_pointers == 0);'
-        # else:
-        #     yield 'GC_all_interior_pointers = 0;'
-        # yield 'boehm_gc_startup_code();'
+    def gc_startup_code(self):
+        s = list(super(QcgcFrameworkGcPolicy, self).gc_startup_code())
+        for i in s:
+            yield i
+        yield 'qcgc_initialize();'
 
 name_to_gcpolicy = {
     'boehm': BoehmGcPolicy,

From pypy.commits at gmail.com  Fri Aug 19 11:19:25 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 19 Aug 2016 08:19:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Allow lone surrogates in _pypyjson
Message-ID: <57b7237d.81a2c20a.9de76.904b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86321:a4b31c94bd5c
Date: 2016-08-19 16:18 +0100
http://bitbucket.org/pypy/pypy/changeset/a4b31c94bd5c/

Log:	Allow lone surrogates in _pypyjson

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -327,7 +327,8 @@
             i += 1
             if ch == '"':
                 content_utf8 = builder.build()
-                content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
+                content_unicode = unicodehelper.decode_utf8(
+                    self.space, content_utf8, allow_surrogates=True)
                 self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.wrap(content_unicode)
@@ -374,7 +375,8 @@
                    # this point
         #
         uchr = runicode.code_to_unichr(val)     # may be a surrogate pair again
-        utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
+        utf8_ch = unicodehelper.encode_utf8(
+            self.space, uchr, allow_surrogates=True)
         builder.append(utf8_ch)
         return i
 
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -10,10 +10,10 @@
     assert dec.skip_whitespace(8) == len(s)
     dec.close()
 
-    
+
 
 class AppTest(object):
-    spaceconfig = {"objspace.usemodules._pypyjson": True}
+    spaceconfig = {"usemodules": ['_pypyjson']}
 
     def test_raise_on_bytes(self):
         import _pypyjson
@@ -40,7 +40,7 @@
         raises(ValueError, _pypyjson.loads, 'fa')
         raises(ValueError, _pypyjson.loads, 'f')
         raises(ValueError, _pypyjson.loads, 'falXX')
-        
+
 
     def test_decode_string(self):
         import _pypyjson
@@ -69,7 +69,7 @@
         import _pypyjson
         assert _pypyjson.loads(r'"\\"') == '\\'
         assert _pypyjson.loads(r'"\""') == '"'
-        assert _pypyjson.loads(r'"\/"') == '/'       
+        assert _pypyjson.loads(r'"\/"') == '/'
         assert _pypyjson.loads(r'"\b"') == '\b'
         assert _pypyjson.loads(r'"\f"') == '\f'
         assert _pypyjson.loads(r'"\n"') == '\n'
@@ -85,7 +85,7 @@
         import _pypyjson
         s = r'"hello\nworld' # missing the trailing "
         raises(ValueError, "_pypyjson.loads(s)")
-        
+
     def test_escape_sequence_unicode(self):
         import _pypyjson
         s = r'"\u1234"'
@@ -166,7 +166,7 @@
     def test_decode_object_nonstring_key(self):
         import _pypyjson
         raises(ValueError, "_pypyjson.loads('{42: 43}')")
-        
+
     def test_decode_array(self):
         import _pypyjson
         assert _pypyjson.loads('[]') == []
@@ -183,7 +183,7 @@
         res = _pypyjson.loads('"z\\ud834\\udd20x"')
         assert res == expected
 
-    def test_surrogate_pair(self):
+    def test_lone_surrogate(self):
         import _pypyjson
         json = '{"a":"\\uD83D"}'
         res = _pypyjson.loads(json)

From pypy.commits at gmail.com  Fri Aug 19 12:23:38 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 19 Aug 2016 09:23:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Backout 7b97d4152ebc: pypy behaves like
 CPython wrt. .pyo now
Message-ID: <57b7328a.4152c20a.4c92.a840@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86322:e9c93d18cbd4
Date: 2016-08-19 17:22 +0100
http://bitbucket.org/pypy/pypy/changeset/e9c93d18cbd4/

Log:	Backout 7b97d4152ebc: pypy behaves like CPython wrt. .pyo now

diff --git a/lib-python/3/test/test_compileall.py b/lib-python/3/test/test_compileall.py
--- a/lib-python/3/test/test_compileall.py
+++ b/lib-python/3/test/test_compileall.py
@@ -202,11 +202,10 @@
 
     # Ensure that the default behavior of compileall's CLI is to create
     # PEP 3147 pyc/pyo files.
-    _pyo = 'pyo' if support.check_impl_detail() else 'pyc'
     for name, ext, switch in [
         ('normal', 'pyc', []),
-        ('optimize', _pyo, ['-O']),
-        ('doubleoptimize', _pyo, ['-OO']),
+        ('optimize', 'pyo', ['-O']),
+        ('doubleoptimize', 'pyo', ['-OO']),
     ]:
         def f(self, ext=ext, switch=switch):
             script_helper.assert_python_ok(*(switch +

From pypy.commits at gmail.com  Fri Aug 19 13:08:36 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Fri, 19 Aug 2016 10:08:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Change const.value to const.obj in
 test_obj because Const in AST changed attribute name
Message-ID: <57b73d14.436ec20a.e855f.b94b@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86323:147982173b0f
Date: 2016-08-19 19:08 +0200
http://bitbucket.org/pypy/pypy/changeset/147982173b0f/

Log:	Change const.value to const.obj in test_obj because Const in AST
	changed attribute name

diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -118,9 +118,9 @@
     def test_object(self):
         ast = self.ast
         const = ast.Const(4)
-        assert const.value == 4
-        const.value = 5
-        assert const.value == 5
+        assert const.obj == 4
+        const.obj = 5
+        assert const.obj == 5
 
     def test_optional(self):
         mod = self.get_ast("x(32)", "eval")

From pypy.commits at gmail.com  Fri Aug 19 14:14:05 2016
From: pypy.commits at gmail.com (wlav)
Date: Fri, 19 Aug 2016 11:14:05 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: force deserialization of global
 functions
Message-ID: <57b74c6d.262ec20a.11a57.c9d0@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86324:741cd54aa263
Date: 2016-08-18 16:40 -0700
http://bitbucket.org/pypy/pypy/changeset/741cd54aa263/

Log:	force deserialization of global functions

diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -716,8 +716,13 @@
 {
    std::vector< TCppMethod_t > methods;
    if ( scope == GLOBAL_HANDLE ) {
+      // TODO: figure out a way of being conservative with reloading
       TCollection* funcs = gROOT->GetListOfGlobalFunctions( kTRUE );
 
+      // tickle deserialization
+      if ( !funcs->FindObject( name.c_str() ) )
+         return methods;
+
       TIter ifunc(funcs);
       TFunction* func = 0;
       while ( (func = (TFunction*)ifunc.Next()) ) {
@@ -1327,6 +1332,11 @@
         }
     } else if (scope == (cppyy_scope_t)GLOBAL_HANDLE) {
         TCollection* funcs = gROOT->GetListOfGlobalFunctions(kTRUE);
+
+        // tickle deserialization
+        if (!funcs->FindObject(name))
+            return (cppyy_index_t*)nullptr;
+
         TFunction* func = 0;
         TIter ifunc(funcs);
         while ((func = (TFunction*)ifunc.Next())) {
@@ -1336,7 +1346,7 @@
     }
 
     if (result.empty())
-        return (cppyy_index_t*)0;
+        return (cppyy_index_t*)nullptr;
 
     cppyy_index_t* llresult = (cppyy_index_t*)malloc(sizeof(cppyy_index_t)*(result.size()+1));
     for (int i = 0; i < (int)result.size(); ++i) llresult[i] = result[i];

From pypy.commits at gmail.com  Fri Aug 19 14:14:09 2016
From: pypy.commits at gmail.com (wlav)
Date: Fri, 19 Aug 2016 11:14:09 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: annotation fix
Message-ID: <57b74c71.c19d1c0a.a7ff3.e80e@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86325:67349a4ab510
Date: 2016-08-18 16:41 -0700
http://bitbucket.org/pypy/pypy/changeset/67349a4ab510/

Log:	annotation fix

diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -534,8 +534,8 @@
     return pystr
 
 def c_charp2stdstring(space, svalue, sz):
-    return _cdata_to_cobject(
-        space, call_capi(space, 'charp2stdstring', [_Arg(s=svalue), _Arg(h=sz)]))
+    return _cdata_to_cobject(space, call_capi(space, 'charp2stdstring',
+        [_Arg(s=svalue), _Arg(h=rffi.cast(rffi.ULONG, sz))]))
 def c_stdstring2charp(space, cppstr):
     sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
     try:

From pypy.commits at gmail.com  Fri Aug 19 14:35:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 19 Aug 2016 11:35:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: I *think* this expresses the logic that we
 want more cleanly
Message-ID: <57b75160.c4ebc20a.fb1e3.d269@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86326:fc46cb0e02ed
Date: 2016-08-19 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/fc46cb0e02ed/

Log:	I *think* this expresses the logic that we want more cleanly

diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -19,6 +19,7 @@
         """ NOT_RPYTHON """
         Module.__init__(self, space, w_name)
         self.lazy = True
+        self.lazy_initial_values_w = {}
         self.__class__.buildloaders()
         self.loaders = self.loaders.copy()    # copy from the class to the inst
         self.submodules_w = []
@@ -57,22 +58,11 @@
             if not self.lazy and self.w_initialdict is None:
                 self.save_module_content_for_future_reload()
 
-    def save_module_content_for_future_reload(self, save_all=False):
-        # Because setdictvalue is unable to immediately load all attributes
-        # (due to an importlib bootstrapping problem), this method needs to be
-        # able to support saving the content of a module's dict without
-        # requiring that the entire dict already be loaded. To support that
-        # properly, when updating the dict, we must be careful to never
-        # overwrite the value of a key already in w_initialdict. (So as to avoid
-        # overriding the builtin value with a user-provided value)
-        if self.space.is_none(self.w_initialdict) or save_all:
-            self.w_initialdict = self.space.call_method(self.w_dict, 'copy')
-        else:
-            w_items = self.space.call_method(self.w_dict, 'items')
-            for w_item in self.space.iteriterable(w_items):
-                w_key, w_value = self.space.fixedview(w_item, expected_length=2)
-                if not self.space.contains_w(self.w_initialdict, w_key):
-                    self.space.setitem(self.w_initialdict, w_key, w_value)
+    def save_module_content_for_future_reload(self):
+        # Save the current dictionary in w_initialdict, for future
+        # reloads.  This forces the dictionary if needed.
+        w_dict = self.getdict(self.space)
+        self.w_initialdict = self.space.call_method(w_dict, 'copy')
 
     @classmethod
     def get_applevel_name(cls):
@@ -101,9 +91,13 @@
         return w_value
 
     def setdictvalue(self, space, attr, w_value):
-        if self.lazy:
-            self._load_lazily(space, attr)
-            self.save_module_content_for_future_reload()
+        if self.lazy and attr not in self.lazy_initial_values_w:
+            # in lazy mode, the first time an attribute changes,
+            # we save away the old (initial) value.  This allows
+            # a future getdict() call to build the correct
+            # self.w_initialdict, containing the initial value.
+            w_initial_value = self._load_lazily(space, attr)
+            self.lazy_initial_values_w[attr] = w_initial_value
         space.setitem_str(self.w_dict, attr, w_value)
         return True
 
@@ -137,11 +131,23 @@
 
     def getdict(self, space):
         if self.lazy:
+            # Force the dictionary by calling all lazy loaders now.
+            # This also saves in self.w_initialdict a copy of all the
+            # initial values, including if they have already been
+            # modified by setdictvalue().
             for name in self.loaders:
                 w_value = self.get(name)
                 space.setitem(self.w_dict, space.new_interned_str(name), w_value)
             self.lazy = False
             self.save_module_content_for_future_reload()
+            for key, w_initial_value in self.lazy_initial_values_w.items():
+                w_key = space.new_interned_str(key)
+                if w_initial_value is not None:
+                    space.setitem(self.w_initialdict, w_key, w_initial_value)
+                else:
+                    if space.finditem(self.w_initialdict, w_key) is not None:
+                        space.delitem(self.w_initialdict, w_key)
+            del self.lazy_initial_values_w
         return self.w_dict
 
     def _cleanup_(self):
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -131,7 +131,7 @@
 
 @unwrap_spec(w_module=MixedModule)
 def save_module_content_for_future_reload(space, w_module):
-    w_module.save_module_content_for_future_reload(save_all=True)
+    w_module.save_module_content_for_future_reload()
 
 def set_code_callback(space, w_callable):
     cache = space.fromcache(CodeHookCache)

From pypy.commits at gmail.com  Fri Aug 19 14:35:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 19 Aug 2016 11:35:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Load sys.filesystemencoding and the
 app-level sys.std{in, out, err}
Message-ID: <57b75162.4fd71c0a.fa5da.eb78@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86327:26dfa2f67afd
Date: 2016-08-19 10:24 +0200
http://bitbucket.org/pypy/pypy/changeset/26dfa2f67afd/

Log:	Load sys.filesystemencoding and the app-level sys.std{in,out,err}
	lazily. This gives a big reduction over the time it takes to build a
	space.

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -14,6 +14,13 @@
         """NOT_RPYTHON""" # because parent __init__ isn't
         if space.config.translating:
             del self.__class__.interpleveldefs['pypy_getudir']
+            del self.__class__.appleveldefs['stdin']
+            del self.__class__.appleveldefs['__stdin__']
+            del self.__class__.appleveldefs['stdout']
+            del self.__class__.appleveldefs['__stdout__']
+            del self.__class__.appleveldefs['stderr']
+            del self.__class__.appleveldefs['__stderr__']
+
         super(Module, self).__init__(space, w_name)
         self.recursionlimit = 100
         self.defaultencoding = "utf-8"
@@ -99,6 +106,15 @@
         'flags'                 : 'app.null_sysflags',
         '_xoptions'             : 'app.null__xoptions',
         'implementation'        : 'app.implementation',
+
+        # these six attributes are here only during tests;
+        # they are removed before translation
+        'stdin'                 : 'std_test.stdin',
+        '__stdin__'             : 'std_test.stdin',
+        'stdout'                : 'std_test.stdout',
+        '__stdout__'            : 'std_test.stdout',
+        'stderr'                : 'std_test.stderr',
+        '__stderr__'            : 'std_test.stderr',
     }
 
     def startup(self, space):
@@ -123,28 +139,12 @@
         space = self.space
 
         if not space.config.translating:
-            from pypy.module.sys.interp_encoding import _getfilesystemencoding
-            self.filesystemencoding = _getfilesystemencoding(space)
-
-        if not space.config.translating:
-            # Install standard streams for tests that don't call app_main.
-            # Always use line buffering, even for tests that capture
-            # standard descriptors.
-            space.appexec([], """():
-                import sys, io
-                sys.stdin = sys.__stdin__ = io.open(0, "r", encoding="ascii",
-                                                    closefd=False)
-                sys.stdin.buffer.raw.name = "<stdin>"
-                sys.stdout = sys.__stdout__ = io.open(1, "w", encoding="ascii",
-                                                      buffering=1,
-                                                      closefd=False)
-                sys.stdout.buffer.raw.name = "<stdout>"
-                sys.stderr = sys.__stderr__ = io.open(2, "w", encoding="ascii",
-                                                      errors="backslashreplace",
-                                                      buffering=1,
-                                                      closefd=False)
-                sys.stderr.buffer.raw.name = "<stderr>"
-               """)
+            ##from pypy.module.sys.interp_encoding import _getfilesystemencoding
+            ##self.filesystemencoding = _getfilesystemencoding(space)
+            # XXX the two lines above take a few seconds to run whenever
+            # we initialize the space; for tests, use a simpler version
+            from pypy.module.sys.interp_encoding import base_encoding
+            self.filesystemencoding = space.wrap(base_encoding)
 
     def flush_std_files(self, space):
         w_stdout = space.sys.getdictvalue(space, 'stdout')
diff --git a/pypy/module/sys/std_test.py b/pypy/module/sys/std_test.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/std_test.py
@@ -0,0 +1,19 @@
+# Install standard streams for tests that don't call app_main.  Always
+# use line buffering, even for tests that capture standard descriptors.
+
+import io
+
+stdin = io.open(0, "r", encoding="ascii",
+                closefd=False)
+stdin.buffer.raw.name = "<stdin>"
+
+stdout = io.open(1, "w", encoding="ascii",
+                 buffering=1,
+                 closefd=False)
+stdout.buffer.raw.name = "<stdout>"
+
+stderr = io.open(2, "w", encoding="ascii",
+                 errors="backslashreplace",
+                 buffering=1,
+                 closefd=False)
+stderr.buffer.raw.name = "<stderr>"

From pypy.commits at gmail.com  Fri Aug 19 14:35:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 19 Aug 2016 11:35:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Another important speed-up for space
 initialization: instead of
Message-ID: <57b75163.c1e31c0a.272c6.2000@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86328:1c4e34c49a4f
Date: 2016-08-19 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/1c4e34c49a4f/

Log:	Another important speed-up for space initialization: instead of
	compiling importlib/_bootstrap.py every time, marshal the bytecode
	and cache it in rpython/_cache/.

diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -20,11 +20,10 @@
         # "from importlib/_boostrap.py import *"
         # It's not a plain "import importlib._boostrap", because we
         # don't want to freeze importlib.__init__.
-        ec = space.getexecutioncontext()
         with open(os.path.join(lib_python, 'importlib', '_bootstrap.py')) as fp:
             source = fp.read()
         pathname = "<frozen importlib._bootstrap>"
-        code_w = ec.compiler.compile(source, pathname, 'exec', 0)
+        code_w = self._cached_compile(source, pathname, 'exec', 0)
         space.setitem(self.w_dict, space.wrap('__name__'), self.w_name)
         space.setitem(self.w_dict, space.wrap('__builtins__'),
                       space.wrap(space.builtin))
@@ -32,6 +31,31 @@
 
         self.w_import = space.wrap(interp_import.import_with_frames_removed)
 
+    def _cached_compile(self, source, *args):
+        from rpython.config.translationoption import CACHE_DIR
+        from pypy.module.marshal import interp_marshal
+
+        space = self.space
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_bootstrap')
+        try:
+            if space.config.translating:
+                raise IOError("don't use the cache when translating pypy")
+            with open(cachename, 'rb') as f:
+                previous = f.read(len(source) + 1)
+                if previous != source + '\x00':
+                    raise IOError("source changed")
+                w_bin = space.newbytes(f.read())
+                code_w = interp_marshal.loads(space, w_bin)
+        except IOError:
+            # must (re)compile the source
+            ec = space.getexecutioncontext()
+            code_w = ec.compiler.compile(source, *args)
+            w_bin = interp_marshal.dumps(space, code_w, space.wrap(2))
+            content = source + '\x00' + space.bytes_w(w_bin)
+            with open(cachename, 'wb') as f:
+                f.write(content)
+        return code_w
+
     def startup(self, space):
         """Copy our __import__ to builtins."""
         w_install = self.getdictvalue(space, '_install')

From pypy.commits at gmail.com  Fri Aug 19 14:35:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 19 Aug 2016 11:35:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: A branch to implement
 kwonly specification for builtin functions
Message-ID: <57b75165.11051c0a.d4d95.edc9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86329:7cbe8c3c91b1
Date: 2016-08-19 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/7cbe8c3c91b1/

Log:	A branch to implement kwonly specification for builtin functions


From pypy.commits at gmail.com  Fri Aug 19 14:35:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 19 Aug 2016 11:35:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: in-progress
Message-ID: <57b75168.a6a5c20a.d74e.d1e7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86330:bb3a95ecfdbf
Date: 2016-08-19 19:10 +0200
http://bitbucket.org/pypy/pypy/changeset/bb3a95ecfdbf/

Log:	in-progress

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -663,7 +663,8 @@
     def __init__(self, func):
         assert isinstance(func, Function)
         Function.__init__(self, func.space, func.code, func.w_func_globals,
-                          func.defs_w, None, func.closure, None, func.name)
+                          func.defs_w, func.w_kw_defs, func.closure,
+                          None, func.name)
         self.w_doc = func.w_doc
         self.w_func_dict = func.w_func_dict
         self.w_module = func.w_module
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -44,12 +44,21 @@
         self.argnames = argnames
         self.varargname = varargname
         self.kwargname = kwargname
+        self.kwonlyargnames = None
 
     def append(self, argname):
-        self.argnames.append(argname)
+        if self.kwonlyargnames is None:
+            self.argnames.append(argname)
+        else:
+            self.kwonlyargnames.append(argname)
+
+    def marker_kwonly(self):
+        assert self.kwonlyargnames is None
+        self.kwonlyargnames = []
 
     def signature(self):
-        return Signature(self.argnames, self.varargname, self.kwargname)
+        return Signature(self.argnames, self.varargname, self.kwargname,
+                         self.kwonlyargnames)
 
 #________________________________________________________________
 
@@ -66,13 +75,6 @@
         """NOT_RPYTHON"""
         raise NotImplementedError
 
-def kwonly(arg_unwrapper):
-    """Mark argument as keyword-only.
-
-    XXX: has no actual effect for now.
-    """
-    return arg_unwrapper
-
 
 class UnwrapSpecRecipe(object):
     "NOT_RPYTHON"
@@ -229,6 +231,11 @@
         name = int_unwrapping_space_method(typ)
         self.checked_space_method(name, app_sig)
 
+    def visit_kwonly(self, _, app_sig):
+        argname = self.orig_arg()
+        assert argname == '__kwonly__'
+        app_sig.marker_kwonly()
+
 
 class UnwrapSpec_EmitRun(UnwrapSpecEmit):
 
@@ -316,6 +323,9 @@
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
+    def visit_kwonly(self, typ):
+        self.run_args.append("None")
+
     def _make_unwrap_activation_class(self, unwrap_spec, cache={}):
         try:
             key = tuple(unwrap_spec)
@@ -468,6 +478,9 @@
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
+    def visit_kwonly(self, typ):
+        self.unwrap.append("None")
+
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
         unwrap_info.apply_over(unwrap_spec)
@@ -563,6 +576,8 @@
                 unwrap_spec.append('args_w')
             elif argname.startswith('w_'):
                 unwrap_spec.append(W_Root)
+            elif argname == '__kwonly__':
+                unwrap_spec.append('kwonly')
             else:
                 unwrap_spec.append(None)
 
@@ -616,6 +631,8 @@
         argnames = sig.argnames
         varargname = sig.varargname
         kwargname = sig.kwargname
+        if sig.kwonlyargnames:
+            import pdb; pdb.set_trace()
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -961,6 +978,8 @@
     def _getdefaults(self, space):
         "NOT_RPYTHON"
         defs_w = []
+        if self._code.sig.kwonlyargnames:
+            import pdb; pdb.set_trace()
         unwrap_spec = self._code._unwrap_spec[-len(self._staticdefs):]
         for i, (name, defaultval) in enumerate(self._staticdefs):
             if name.startswith('w_'):
@@ -977,7 +996,9 @@
                     defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
             UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
+            allargnames = (self._code.sig.argnames +
+                           self._code.sig.kwonlyargnames)
+            alldefs_w = [UNDEFINED] * len(allargnames)
             if defs_w:
                 alldefs_w[-len(defs_w):] = defs_w
             code = self._code
@@ -998,7 +1019,7 @@
                     assert isinstance(w_default, W_Root)
                     assert argname.startswith('w_')
                     argname = argname[2:]
-                    j = self._code.sig.argnames.index(argname)
+                    j = allargnames.index(argname)
                     assert alldefs_w[j] in (UNDEFINED, None)
                     alldefs_w[j] = w_default
             first_defined = 0
@@ -1027,9 +1048,11 @@
     def build(cache, gateway):
         "NOT_RPYTHON"
         space = cache.space
-        defs = gateway._getdefaults(space) # needs to be implemented by subclass
+        defs = gateway._getdefaults(space)
         code = gateway._code
-        fn = FunctionWithFixedCode(space, code, None, defs, forcename=gateway.name)
+        w_kw_defs = None #XXXXXXXXXXXXXXXXXXX
+        fn = FunctionWithFixedCode(space, code, None, defs, w_kw_defs,
+                                   forcename=gateway.name)
         if not space.config.translating:
             fn.add_to_table()
         if gateway.as_classmethod:
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -39,6 +39,7 @@
 
     def scope_length(self):
         scopelen = len(self.argnames)
+        scopelen += len(self.kwonlyargnames)
         scopelen += self.has_vararg()
         scopelen += self.has_kwarg()
         return scopelen
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -47,6 +47,12 @@
         code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace, "index"])
         assert code.signature() == Signature(["index"], None, None)
 
+        def f(space, __kwonly__, w_x):
+            pass
+        code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace,
+                                                   "kwonly", W_Root])
+        assert code.signature() == Signature([], kwonlyargnames=['x'])
+
 
     def test_call(self):
         def c(space, w_x, w_y, hello_w):
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -15,8 +15,7 @@
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
 
-from pypy.interpreter.gateway import (
-    unwrap_spec, WrappedDefault, Unwrapper, kwonly)
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, Unwrapper
 from pypy.interpreter.error import (
     OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror)
 from pypy.interpreter.executioncontext import ExecutionContext
@@ -211,7 +210,8 @@
             "%s: %s unavailable on this platform", funcname, arg)
 
 @unwrap_spec(flags=c_int, mode=c_int, dir_fd=DirFD(rposix.HAVE_OPENAT))
-def open(space, w_path, flags, mode=0777, dir_fd=DEFAULT_DIR_FD):
+def open(space, w_path, flags, mode=0777,
+         __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """open(path, flags, mode=0o777, *, dir_fd=None)
 
 Open a file for low level IO.  Returns a file handle (integer).
@@ -428,8 +428,8 @@
 @unwrap_spec(
     path=path_or_fd(allow_fd=True),
     dir_fd=DirFD(rposix.HAVE_FSTATAT),
-    follow_symlinks=kwonly(bool))
-def stat(space, path, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
+    follow_symlinks=bool)
+def stat(space, path, __kwonly__, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
     """stat(path, *, dir_fd=None, follow_symlinks=True) -> stat result
 
 Perform a stat system call on the given path.
@@ -476,7 +476,7 @@
 @unwrap_spec(
     path=path_or_fd(allow_fd=False),
     dir_fd=DirFD(rposix.HAVE_FSTATAT))
-def lstat(space, path, dir_fd=DEFAULT_DIR_FD):
+def lstat(space, path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """lstat(path, *, dir_fd=None) -> stat result
 
 Like stat(), but do not follow symbolic links.
@@ -551,9 +551,9 @@
         raise wrap_oserror(space, e)
 
 @unwrap_spec(mode=c_int,
-    dir_fd=DirFD(rposix.HAVE_FACCESSAT), effective_ids=kwonly(bool),
-    follow_symlinks=kwonly(bool))
-def access(space, w_path, mode,
+    dir_fd=DirFD(rposix.HAVE_FACCESSAT), effective_ids=bool,
+    follow_symlinks=bool)
+def access(space, w_path, mode, __kwonly__,
         dir_fd=DEFAULT_DIR_FD, effective_ids=False, follow_symlinks=True):
     """\
 access(path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True)
@@ -626,7 +626,7 @@
         return space.wrap(rc)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
-def unlink(space, w_path, dir_fd=DEFAULT_DIR_FD):
+def unlink(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """unlink(path, *, dir_fd=None)
 
 Remove a file (same as remove()).
@@ -645,7 +645,7 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
-def remove(space, w_path, dir_fd=DEFAULT_DIR_FD):
+def remove(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """remove(path, *, dir_fd=None)
 
 Remove a file (same as unlink()).
@@ -710,7 +710,7 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_MKDIRAT))
-def mkdir(space, w_path, mode=0o777, dir_fd=DEFAULT_DIR_FD):
+def mkdir(space, w_path, mode=0o777, __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """mkdir(path, mode=0o777, *, dir_fd=None)
 
 Create a directory.
@@ -731,7 +731,7 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
-def rmdir(space, w_path, dir_fd=DEFAULT_DIR_FD):
+def rmdir(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """rmdir(path, *, dir_fd=None)
 
 Remove a directory.
@@ -898,8 +898,9 @@
     return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),
-             follow_symlinks=kwonly(bool))
-def chmod(space, w_path, mode, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
+             follow_symlinks=bool)
+def chmod(space, w_path, mode, __kwonly__,
+          dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
     """chmod(path, mode, *, dir_fd=None, follow_symlinks=True)
 
 Change the access permissions of a file.
@@ -965,7 +966,7 @@
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
-def rename(space, w_src, w_dst,
+def rename(space, w_src, w_dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD):
     """rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None)
 
@@ -989,7 +990,7 @@
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
-def replace(space, w_src, w_dst,
+def replace(space, w_src, w_dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD):
     """replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None)
 
@@ -1012,7 +1013,7 @@
         raise wrap_oserror(space, e)
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_MKFIFOAT))
-def mkfifo(space, w_path, mode=0666, dir_fd=DEFAULT_DIR_FD):
+def mkfifo(space, w_path, mode=0666, __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """mkfifo(path, mode=0o666, *, dir_fd=None)
 
 Create a FIFO (a POSIX named pipe).
@@ -1031,7 +1032,8 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(mode=c_int, device=c_int, dir_fd=DirFD(rposix.HAVE_MKNODAT))
-def mknod(space, w_filename, mode=0600, device=0, dir_fd=DEFAULT_DIR_FD):
+def mknod(space, w_filename, mode=0600, device=0,
+          __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """mknod(filename, mode=0o600, device=0, *, dir_fd=None)
 
 Create a filesystem node (file, device special file or named pipe)
@@ -1093,9 +1095,9 @@
 @unwrap_spec(
     src='fsencode', dst='fsencode',
     src_dir_fd=DirFD(rposix.HAVE_LINKAT), dst_dir_fd=DirFD(rposix.HAVE_LINKAT),
-    follow_symlinks=kwonly(bool))
+    follow_symlinks=bool)
 def link(
-        space, src, dst,
+        space, src, dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD,
         follow_symlinks=True):
     """\
@@ -1125,7 +1127,7 @@
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_SYMLINKAT))
 def symlink(space, w_src, w_dst, w_target_is_directory=None,
-        dir_fd=DEFAULT_DIR_FD):
+            __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """symlink(src, dst, target_is_directory=False, *, dir_fd=None)
 
 Create a symbolic link pointing to src named dst.
@@ -1153,7 +1155,7 @@
 @unwrap_spec(
     path=path_or_fd(allow_fd=False),
     dir_fd=DirFD(rposix.HAVE_READLINKAT))
-def readlink(space, path, dir_fd=DEFAULT_DIR_FD):
+def readlink(space, path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """readlink(path, *, dir_fd=None) -> path
 
 Return a string representing the path to which the symbolic link points.
@@ -1353,9 +1355,9 @@
 
 @unwrap_spec(
     path=path_or_fd(allow_fd=rposix.HAVE_FUTIMENS or rposix.HAVE_FUTIMES),
-    w_times=WrappedDefault(None), w_ns=kwonly(WrappedDefault(None)),
-    dir_fd=DirFD(rposix.HAVE_UTIMENSAT), follow_symlinks=kwonly(bool))
-def utime(space, path, w_times, w_ns, dir_fd=DEFAULT_DIR_FD,
+    w_times=WrappedDefault(None), w_ns=WrappedDefault(None),
+    dir_fd=DirFD(rposix.HAVE_UTIMENSAT), follow_symlinks=bool)
+def utime(space, path, w_times, __kwonly__, w_ns, dir_fd=DEFAULT_DIR_FD,
           follow_symlinks=True):
     """utime(path, times=None, *, ns=None, dir_fd=None, follow_symlinks=True)
 
@@ -1889,8 +1891,9 @@
 
 @unwrap_spec(
     uid=c_uid_t, gid=c_gid_t,
-    dir_fd=DirFD(rposix.HAVE_FCHOWNAT), follow_symlinks=kwonly(bool))
-def chown(space, w_path, uid, gid, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
+    dir_fd=DirFD(rposix.HAVE_FCHOWNAT), follow_symlinks=bool)
+def chown(space, w_path, uid, gid, __kwonly__,
+          dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
     """chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True)
 
 Change the owner and group id of path to the numeric uid and gid.

From pypy.commits at gmail.com  Fri Aug 19 14:35:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 19 Aug 2016 11:35:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: merge heads
Message-ID: <57b7516a.c4ebc20a.fb1e3.d277@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86331:47258bd66b35
Date: 2016-08-19 20:34 +0200
http://bitbucket.org/pypy/pypy/changeset/47258bd66b35/

Log:	merge heads

diff --git a/lib-python/3/test/test_compileall.py b/lib-python/3/test/test_compileall.py
--- a/lib-python/3/test/test_compileall.py
+++ b/lib-python/3/test/test_compileall.py
@@ -202,11 +202,10 @@
 
     # Ensure that the default behavior of compileall's CLI is to create
     # PEP 3147 pyc/pyo files.
-    _pyo = 'pyo' if support.check_impl_detail() else 'pyc'
     for name, ext, switch in [
         ('normal', 'pyc', []),
-        ('optimize', _pyo, ['-O']),
-        ('doubleoptimize', _pyo, ['-OO']),
+        ('optimize', 'pyo', ['-O']),
+        ('doubleoptimize', 'pyo', ['-OO']),
     ]:
         def f(self, ext=ext, switch=switch):
             script_helper.assert_python_ok(*(switch +
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -327,7 +327,8 @@
             i += 1
             if ch == '"':
                 content_utf8 = builder.build()
-                content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
+                content_unicode = unicodehelper.decode_utf8(
+                    self.space, content_utf8, allow_surrogates=True)
                 self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.wrap(content_unicode)
@@ -374,7 +375,8 @@
                    # this point
         #
         uchr = runicode.code_to_unichr(val)     # may be a surrogate pair again
-        utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
+        utf8_ch = unicodehelper.encode_utf8(
+            self.space, uchr, allow_surrogates=True)
         builder.append(utf8_ch)
         return i
 
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -10,10 +10,10 @@
     assert dec.skip_whitespace(8) == len(s)
     dec.close()
 
-    
+
 
 class AppTest(object):
-    spaceconfig = {"objspace.usemodules._pypyjson": True}
+    spaceconfig = {"usemodules": ['_pypyjson']}
 
     def test_raise_on_bytes(self):
         import _pypyjson
@@ -40,7 +40,7 @@
         raises(ValueError, _pypyjson.loads, 'fa')
         raises(ValueError, _pypyjson.loads, 'f')
         raises(ValueError, _pypyjson.loads, 'falXX')
-        
+
 
     def test_decode_string(self):
         import _pypyjson
@@ -69,7 +69,7 @@
         import _pypyjson
         assert _pypyjson.loads(r'"\\"') == '\\'
         assert _pypyjson.loads(r'"\""') == '"'
-        assert _pypyjson.loads(r'"\/"') == '/'       
+        assert _pypyjson.loads(r'"\/"') == '/'
         assert _pypyjson.loads(r'"\b"') == '\b'
         assert _pypyjson.loads(r'"\f"') == '\f'
         assert _pypyjson.loads(r'"\n"') == '\n'
@@ -85,7 +85,7 @@
         import _pypyjson
         s = r'"hello\nworld' # missing the trailing "
         raises(ValueError, "_pypyjson.loads(s)")
-        
+
     def test_escape_sequence_unicode(self):
         import _pypyjson
         s = r'"\u1234"'
@@ -166,7 +166,7 @@
     def test_decode_object_nonstring_key(self):
         import _pypyjson
         raises(ValueError, "_pypyjson.loads('{42: 43}')")
-        
+
     def test_decode_array(self):
         import _pypyjson
         assert _pypyjson.loads('[]') == []
@@ -183,7 +183,7 @@
         res = _pypyjson.loads('"z\\ud834\\udd20x"')
         assert res == expected
 
-    def test_surrogate_pair(self):
+    def test_lone_surrogate(self):
         import _pypyjson
         json = '{"a":"\\uD83D"}'
         res = _pypyjson.loads(json)

From pypy.commits at gmail.com  Fri Aug 19 14:37:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 19 Aug 2016 11:37:17 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57b751dd.4317c20a.a6bc8.d789@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r778:525dc29ad26c
Date: 2016-08-19 20:37 +0200
http://bitbucket.org/pypy/pypy.org/changeset/525dc29ad26c/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 61.7
+      value: 61.8
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $64836 of $105000 (61.7%)
+   $64859 of $105000 (61.8%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5314 left
+    we have $5336 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $30870 of $80000 (38.6%)
+   $30875 of $80000 (38.6%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $23524 left
+    we have $23528 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Fri Aug 19 14:45:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 19 Aug 2016 11:45:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: more fixes
Message-ID: <57b753ca.a6a5c20a.d74e.d51e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86332:b130c20feba7
Date: 2016-08-19 20:44 +0200
http://bitbucket.org/pypy/pypy/changeset/b130c20feba7/

Log:	more fixes

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -153,7 +153,7 @@
 
     @jit.unroll_safe
     def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
-                         w_kw_defs=None, blindargs=0):
+                         kw_defs_w=None, blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
         """
@@ -281,19 +281,18 @@
                 else:
                     missing_positional.append(signature.argnames[i])
 
-            # finally, fill kwonly arguments with w_kw_defs (if needed)
-            for i in range(co_argcount, co_argcount + co_kwonlyargcount):
-                if scope_w[i] is not None:
+            # finally, fill kwonly arguments with kw_defs_w (if needed)
+            for i in range(co_kwonlyargcount):
+                j = co_argcount + i
+                if scope_w[j] is not None:
                     continue
-                name = signature.kwonlyargnames[i - co_argcount]
-                if w_kw_defs is None:
+                try:
+                    w_def = signature.get_kwonly_default(i, kw_defs_w)
+                except KeyError:
+                    name = signature.kwonlyargnames[i]
                     missing_kwonly.append(name)
-                    continue
-                w_def = self.space.finditem_str(w_kw_defs, name)
-                if w_def is not None:
-                    scope_w[i] = w_def
                 else:
-                    missing_kwonly.append(name)
+                    scope_w[j] = w_def
 
         if missing_positional:
             raise ArgErrMissing(missing_positional, True)
@@ -303,7 +302,7 @@
 
     def parse_into_scope(self, w_firstarg,
                          scope_w, fnname, signature, defaults_w=None,
-                         w_kw_defs=None):
+                         kw_defs_w=None):
         """Parse args and kwargs to initialize a frame
         according to the signature of code object.
         Store the argumentvalues into scope_w.
@@ -312,30 +311,30 @@
         try:
             self._match_signature(w_firstarg,
                                   scope_w, signature, defaults_w,
-                                  w_kw_defs, 0)
+                                  kw_defs_w, 0)
         except ArgErr as e:
             raise oefmt(self.space.w_TypeError, "%s() %8", fnname, e.getmsg())
         return signature.scope_length()
 
-    def _parse(self, w_firstarg, signature, defaults_w, w_kw_defs, blindargs=0):
+    def _parse(self, w_firstarg, signature, defaults_w, kw_defs_w, blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
         """
         scopelen = signature.scope_length()
         scope_w = [None] * scopelen
         self._match_signature(w_firstarg, scope_w, signature, defaults_w,
-                              w_kw_defs, blindargs)
+                              kw_defs_w, blindargs)
         return scope_w
 
 
     def parse_obj(self, w_firstarg,
-                  fnname, signature, defaults_w=None, w_kw_defs=None,
+                  fnname, signature, defaults_w=None, kw_defs_w=None,
                   blindargs=0):
         """Parse args and kwargs to initialize a frame
         according to the signature of code object.
         """
         try:
-            return self._parse(w_firstarg, signature, defaults_w, w_kw_defs,
+            return self._parse(w_firstarg, signature, defaults_w, kw_defs_w,
                                blindargs)
         except ArgErr as e:
             raise oefmt(self.space.w_TypeError, "%s() %8", fnname, e.getmsg())
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -36,9 +36,9 @@
                           'closure?[*]',
                           'defs_w?[*]',
                           'name?',
-                          'w_kw_defs?']
+                          'kw_defs_w?']
 
-    def __init__(self, space, code, w_globals=None, defs_w=[], w_kw_defs=None,
+    def __init__(self, space, code, w_globals=None, defs_w=[], kw_defs_w=None,
                  closure=None, w_ann=None, forcename=None, qualname=None):
         self.space = space
         self.name = forcename or code.co_name
@@ -48,7 +48,7 @@
         self.w_func_globals = w_globals  # the globals dictionary
         self.closure = closure    # normally, list of Cell instances or None
         self.defs_w = defs_w
-        self.w_kw_defs = w_kw_defs
+        self.kw_defs_w = kw_defs_w
         self.w_func_dict = None # filled out below if needed
         self.w_module = None
         self.w_ann = w_ann
@@ -373,19 +373,35 @@
         self.defs_w = []
 
     def fget_func_kwdefaults(self, space):
-        if self.w_kw_defs is None:
+        if self.kw_defs_w is None:
             return space.w_None
-        return self.w_kw_defs
+        w_result = space.newdict(strdict=True)
+        for key, w_value in self.kw_defs_w.items():
+            space.setitem_str(w_result, key, w_value)
+        return w_result
+
+    @staticmethod
+    def add_kwdefaults(space, kw_defs_w, w_key, w_value):
+        key = space.unicode_w(w_key).encode('utf-8')
+        kw_defs_w[key] = w_value
 
     def fset_func_kwdefaults(self, space, w_new):
         if space.is_w(w_new, space.w_None):
-            w_new = None
+            self.kw_defs_w = None
         elif not space.isinstance_w(w_new, space.w_dict):
             raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
-        self.w_kw_defs = w_new
+        else:
+            # must assign a new dictionary to 'kw_defs_w', never mutate
+            # the existing dictionary: this is because
+            # Signature.get_kwonly_default() is an elidable function.
+            new_w = {}
+            for w_key in space.unpackiterable(w_new):
+                w_value = space.getitem(w_new, w_key)
+                self.add_kwdefaults(space, new_w, w_key, w_value)
+            self.kw_defs_w = new_w
 
     def fdel_func_kwdefaults(self, space):
-        self.w_kw_defs = None
+        self.kw_defs_w = None
 
     def fget_func_doc(self, space):
         if self.w_doc is None:
@@ -663,7 +679,7 @@
     def __init__(self, func):
         assert isinstance(func, Function)
         Function.__init__(self, func.space, func.code, func.w_func_globals,
-                          func.defs_w, func.w_kw_defs, func.closure,
+                          func.defs_w, func.kw_defs_w, func.closure,
                           None, func.name)
         self.w_doc = func.w_doc
         self.w_func_dict = func.w_func_dict
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -723,7 +723,7 @@
         space = func.space
         activation = self.activation
         scope_w = args.parse_obj(w_obj, func.name, self.sig,
-                                 func.defs_w, func.w_kw_defs, self.minargs)
+                                 func.defs_w, func.kw_defs_w, self.minargs)
         try:
             w_result = activation._run(space, scope_w)
         except DescrMismatch:
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -262,7 +262,7 @@
         fresh_frame = jit.hint(frame, access_directly=True,
                                       fresh_virtualizable=True)
         args.parse_into_scope(None, fresh_frame.locals_cells_stack_w, func.name,
-                              sig, func.defs_w, func.w_kw_defs)
+                              sig, func.defs_w, func.kw_defs_w)
         fresh_frame.init_cells()
         return frame.run()
 
@@ -274,7 +274,7 @@
         fresh_frame = jit.hint(frame, access_directly=True,
                                       fresh_virtualizable=True)
         args.parse_into_scope(w_obj, fresh_frame.locals_cells_stack_w, func.name,
-                              sig, func.defs_w, func.w_kw_defs)
+                              sig, func.defs_w, func.kw_defs_w)
         fresh_frame.init_cells()
         return frame.run()
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1235,15 +1235,16 @@
             for i in range(len(names_w) - 1, -1, -1):
                 space.setitem(w_ann, names_w[i], self.popvalue())
         defaultarguments = self.popvalues(posdefaults)
-        w_kw_defs = None
+        kw_defs_w = None
         if kwdefaults:
-            w_kw_defs = space.newdict(strdict=True)
-            for i in range(kwdefaults - 1, -1, -1):
-                w_name = self.popvalue()
-                w_def = self.popvalue()
-                space.setitem(w_kw_defs, w_def, w_name)
+            kw_defs_w = {}
+            for i in range(kwdefaults):
+                w_defvalue = self.popvalue()
+                w_defname = self.popvalue()
+                function.Function.add_kwdefaults(space, kw_defs_w,
+                                                 w_defname, w_defvalue)
         fn = function.Function(space, codeobj, self.get_w_globals(), defaultarguments,
-                               w_kw_defs, freevars, w_ann, qualname=qualname)
+                               kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
     def MAKE_FUNCTION(self, oparg, next_instr):
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -25,6 +25,13 @@
             pass
         return -1
 
+    @jit.elidable
+    def get_kwonly_default(self, i, kw_defs_w):
+        if kw_defs_w is None:
+            raise KeyError
+        name = self.kwonlyargnames[i]
+        return kw_defs_w[name]
+
     def num_argnames(self):
         return len(self.argnames)
 

From pypy.commits at gmail.com  Fri Aug 19 18:27:44 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 19 Aug 2016 15:27:44 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: add test of getitem,
 fix CPyBuffer object so they pass. Add __buffer__ slot to str.
Message-ID: <57b787e0.031dc20a.34dd5.237c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86333:0a85b1b3bb7f
Date: 2016-08-20 10:26 +1200
http://bitbucket.org/pypy/pypy/changeset/0a85b1b3bb7f/

Log:	add test of getitem, fix CPyBuffer object so they pass. Add
	__buffer__ slot to str.

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -9,7 +9,7 @@
     """Return 1 if obj supports the buffer interface otherwise 0."""
     as_buffer = pyobj.c_ob_type.c_tp_as_buffer
     flags = pyobj.c_ob_type.c_tp_flags
-    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.bf_getbuffer):
+    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
         return 1
     return 0  
 
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl, Py_buffer)
+    mangle_name, pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
@@ -301,24 +301,42 @@
     # Similar to Py_buffer
     _immutable_ = True
 
-    def __init__(self, ptr, size, w_obj):
-        # XXX leak of ptr
+    def __init__(self, ptr, size, w_obj, pybuf=None):
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
-        self.readonly = True
+        if pybuf is None:
+            self.format = 'B'
+            self.shape = [size]
+            self.strides = [1]
+            self.ndim = 1
+            self.itemsize = 1
+            self.readonly = True
+        else:
+            self.format = rffi.charp2str(pybuf.c_format)
+            self.ndim = pybuf.c_ndim
+            self.shape = [pybuf.c_shape[i] for i in range(self.ndim)]
+            self.strides = [pybuf.c_strides[i] for i in range(self.ndim)]
+            self.itemsize = pybuf.c_itemsize 
+            self.readonly = pybuf.c_readonly
 
     def getlength(self):
         return self.size
 
     def getitem(self, index):
-        return self.ptr[index]
+        start = index * self.itemsize
+        stop = (index + 1) * self.itemsize
+        return ''.join([self.ptr[i] for i in range(start, stop)])
 
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
 
     def getformat(self):
-        return rffi.charp2str(self.ptr.c_format)
+        return self.format
+
+    def getshape(self):
+        return self.shape
+
 
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
@@ -332,13 +350,15 @@
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
     # XXX leak
-    pybuf = lltype.malloc(Py_buffer, flavor='raw', track_allocation=False)
-    # XXX flags are not in w_args?
-    flags = rffi.cast(rffi.INT_real,0)
-    size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
-    if size < 0:
-        space.fromcache(State).check_and_raise_exception(always=True)
-    return space.newbuffer(CPyBuffer(pybuf, size, w_self))
+    with lltype.scoped_alloc(Py_buffer) as pybuf:
+        # XXX flags are not in w_args?
+        flags = rffi.cast(rffi.INT_real,0)
+        size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+        if size < 0:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        ptr = pybuf.c_buf
+        size = pybuf.c_len
+        return space.newbuffer(CPyBuffer(ptr, size, w_self, pybuf))
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
@@ -504,8 +524,6 @@
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
         api_func = slot_tp_getattro.api_func
-    elif name == 'tp_as_buffer':
-        raise NotImplementedError
     elif name == 'tp_call':
         call_fn = w_type.getdictvalue(space, '__call__')
         if call_fn is None:
@@ -561,6 +579,19 @@
                              w_stararg=w_args, w_starstararg=w_kwds)
             return space.call_args(space.get(new_fn, w_self), args)
         api_func = slot_tp_new.api_func
+    elif name == 'tp_as_buffer.c_bf_getbuffer':
+        buf_fn = w_type.getdictvalue(space, '__buffer__')
+        if buf_fn is None:
+            return
+        @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
+                rffi.INT_real, header=None, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def buff_w(space, w_self, w_args, w_kwds):
+            # XXX this is wrong, needs a test
+            args = Arguments(space, [w_self],
+                             w_stararg=w_args, w_starstararg=w_kwds)
+            return space.call_args(space.get(buff_fn, w_self), args)
+        api_func = buff_w.api_func
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
@@ -881,6 +912,7 @@
 
 # partial sort to solve some slot conflicts:
 # Number slots before Mapping slots before Sequence slots.
+# also prefer the new buffer interface
 # These are the only conflicts between __name__ methods
 def slotdef_sort_key(slotdef):
     if slotdef.slot_name.startswith('tp_as_number'):
@@ -889,6 +921,10 @@
         return 2
     if slotdef.slot_name.startswith('tp_as_sequence'):
         return 3
+    if slodef.slot_name == 'tp_as_buffer.c_bf_getbuffer':
+        return 100
+    if slodef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer':
+        return 101
     return 0
 slotdefs = sorted(slotdefs, key=slotdef_sort_key)
 
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -3,7 +3,6 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 
-
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
         if space.is_true(space.lt(space.sys.get('version_info'),
@@ -13,14 +12,20 @@
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
+        w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
+        assert w_char == space.wrap('h')
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):
+        import struct
         module = self.import_module(name='buffer_test')
         arr = module.PyMyArray(10)
         y = memoryview(arr)
         assert y.format == 'i'
+        s = y[3]
+        assert len(s) == struct.calcsize('i')
+        assert s == struct.pack('i', 3)
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -17,7 +17,8 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr)
+    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -608,6 +609,7 @@
                 bf_getwritebuffer.api_func.get_wrapper(space))
     pto.c_tp_as_buffer = c_buf
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
+    pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def type_dealloc(space, obj):
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -464,6 +464,9 @@
         raise oefmt(space.w_TypeError,
                     "Cannot use string as modifiable buffer")
 
+    def descr_getbuffer(self, space):
+        return self
+
     charbuf_w = str_w
 
     def listview_bytes(self):
@@ -925,6 +928,7 @@
     translate = interpindirect2app(W_AbstractBytesObject.descr_translate),
     upper = interpindirect2app(W_AbstractBytesObject.descr_upper),
     zfill = interpindirect2app(W_AbstractBytesObject.descr_zfill),
+    __buffer__ = interp2app(W_BytesObject.descr_getbuffer),
 
     format = interpindirect2app(W_BytesObject.descr_format),
     __format__ = interpindirect2app(W_BytesObject.descr__format__),

From pypy.commits at gmail.com  Sat Aug 20 02:45:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 19 Aug 2016 23:45:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: Backed out changeset
 b130c20feba7
Message-ID: <57b7fc75.87941c0a.620ef.a204@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86334:2c59ae270f5a
Date: 2016-08-20 08:44 +0200
http://bitbucket.org/pypy/pypy/changeset/2c59ae270f5a/

Log:	Backed out changeset b130c20feba7

	Not so easy: it wouldn't support ``f.__kwdefaults__['a']=5`` or
	logic like ``f.__kwdefaults__=d={}; d['a']=5``.

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -153,7 +153,7 @@
 
     @jit.unroll_safe
     def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
-                         kw_defs_w=None, blindargs=0):
+                         w_kw_defs=None, blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
         """
@@ -281,18 +281,19 @@
                 else:
                     missing_positional.append(signature.argnames[i])
 
-            # finally, fill kwonly arguments with kw_defs_w (if needed)
-            for i in range(co_kwonlyargcount):
-                j = co_argcount + i
-                if scope_w[j] is not None:
+            # finally, fill kwonly arguments with w_kw_defs (if needed)
+            for i in range(co_argcount, co_argcount + co_kwonlyargcount):
+                if scope_w[i] is not None:
                     continue
-                try:
-                    w_def = signature.get_kwonly_default(i, kw_defs_w)
-                except KeyError:
-                    name = signature.kwonlyargnames[i]
+                name = signature.kwonlyargnames[i - co_argcount]
+                if w_kw_defs is None:
                     missing_kwonly.append(name)
+                    continue
+                w_def = self.space.finditem_str(w_kw_defs, name)
+                if w_def is not None:
+                    scope_w[i] = w_def
                 else:
-                    scope_w[j] = w_def
+                    missing_kwonly.append(name)
 
         if missing_positional:
             raise ArgErrMissing(missing_positional, True)
@@ -302,7 +303,7 @@
 
     def parse_into_scope(self, w_firstarg,
                          scope_w, fnname, signature, defaults_w=None,
-                         kw_defs_w=None):
+                         w_kw_defs=None):
         """Parse args and kwargs to initialize a frame
         according to the signature of code object.
         Store the argumentvalues into scope_w.
@@ -311,30 +312,30 @@
         try:
             self._match_signature(w_firstarg,
                                   scope_w, signature, defaults_w,
-                                  kw_defs_w, 0)
+                                  w_kw_defs, 0)
         except ArgErr as e:
             raise oefmt(self.space.w_TypeError, "%s() %8", fnname, e.getmsg())
         return signature.scope_length()
 
-    def _parse(self, w_firstarg, signature, defaults_w, kw_defs_w, blindargs=0):
+    def _parse(self, w_firstarg, signature, defaults_w, w_kw_defs, blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
         """
         scopelen = signature.scope_length()
         scope_w = [None] * scopelen
         self._match_signature(w_firstarg, scope_w, signature, defaults_w,
-                              kw_defs_w, blindargs)
+                              w_kw_defs, blindargs)
         return scope_w
 
 
     def parse_obj(self, w_firstarg,
-                  fnname, signature, defaults_w=None, kw_defs_w=None,
+                  fnname, signature, defaults_w=None, w_kw_defs=None,
                   blindargs=0):
         """Parse args and kwargs to initialize a frame
         according to the signature of code object.
         """
         try:
-            return self._parse(w_firstarg, signature, defaults_w, kw_defs_w,
+            return self._parse(w_firstarg, signature, defaults_w, w_kw_defs,
                                blindargs)
         except ArgErr as e:
             raise oefmt(self.space.w_TypeError, "%s() %8", fnname, e.getmsg())
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -36,9 +36,9 @@
                           'closure?[*]',
                           'defs_w?[*]',
                           'name?',
-                          'kw_defs_w?']
+                          'w_kw_defs?']
 
-    def __init__(self, space, code, w_globals=None, defs_w=[], kw_defs_w=None,
+    def __init__(self, space, code, w_globals=None, defs_w=[], w_kw_defs=None,
                  closure=None, w_ann=None, forcename=None, qualname=None):
         self.space = space
         self.name = forcename or code.co_name
@@ -48,7 +48,7 @@
         self.w_func_globals = w_globals  # the globals dictionary
         self.closure = closure    # normally, list of Cell instances or None
         self.defs_w = defs_w
-        self.kw_defs_w = kw_defs_w
+        self.w_kw_defs = w_kw_defs
         self.w_func_dict = None # filled out below if needed
         self.w_module = None
         self.w_ann = w_ann
@@ -373,35 +373,19 @@
         self.defs_w = []
 
     def fget_func_kwdefaults(self, space):
-        if self.kw_defs_w is None:
+        if self.w_kw_defs is None:
             return space.w_None
-        w_result = space.newdict(strdict=True)
-        for key, w_value in self.kw_defs_w.items():
-            space.setitem_str(w_result, key, w_value)
-        return w_result
-
-    @staticmethod
-    def add_kwdefaults(space, kw_defs_w, w_key, w_value):
-        key = space.unicode_w(w_key).encode('utf-8')
-        kw_defs_w[key] = w_value
+        return self.w_kw_defs
 
     def fset_func_kwdefaults(self, space, w_new):
         if space.is_w(w_new, space.w_None):
-            self.kw_defs_w = None
+            w_new = None
         elif not space.isinstance_w(w_new, space.w_dict):
             raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
-        else:
-            # must assign a new dictionary to 'kw_defs_w', never mutate
-            # the existing dictionary: this is because
-            # Signature.get_kwonly_default() is an elidable function.
-            new_w = {}
-            for w_key in space.unpackiterable(w_new):
-                w_value = space.getitem(w_new, w_key)
-                self.add_kwdefaults(space, new_w, w_key, w_value)
-            self.kw_defs_w = new_w
+        self.w_kw_defs = w_new
 
     def fdel_func_kwdefaults(self, space):
-        self.kw_defs_w = None
+        self.w_kw_defs = None
 
     def fget_func_doc(self, space):
         if self.w_doc is None:
@@ -679,7 +663,7 @@
     def __init__(self, func):
         assert isinstance(func, Function)
         Function.__init__(self, func.space, func.code, func.w_func_globals,
-                          func.defs_w, func.kw_defs_w, func.closure,
+                          func.defs_w, func.w_kw_defs, func.closure,
                           None, func.name)
         self.w_doc = func.w_doc
         self.w_func_dict = func.w_func_dict
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -723,7 +723,7 @@
         space = func.space
         activation = self.activation
         scope_w = args.parse_obj(w_obj, func.name, self.sig,
-                                 func.defs_w, func.kw_defs_w, self.minargs)
+                                 func.defs_w, func.w_kw_defs, self.minargs)
         try:
             w_result = activation._run(space, scope_w)
         except DescrMismatch:
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -262,7 +262,7 @@
         fresh_frame = jit.hint(frame, access_directly=True,
                                       fresh_virtualizable=True)
         args.parse_into_scope(None, fresh_frame.locals_cells_stack_w, func.name,
-                              sig, func.defs_w, func.kw_defs_w)
+                              sig, func.defs_w, func.w_kw_defs)
         fresh_frame.init_cells()
         return frame.run()
 
@@ -274,7 +274,7 @@
         fresh_frame = jit.hint(frame, access_directly=True,
                                       fresh_virtualizable=True)
         args.parse_into_scope(w_obj, fresh_frame.locals_cells_stack_w, func.name,
-                              sig, func.defs_w, func.kw_defs_w)
+                              sig, func.defs_w, func.w_kw_defs)
         fresh_frame.init_cells()
         return frame.run()
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1235,16 +1235,15 @@
             for i in range(len(names_w) - 1, -1, -1):
                 space.setitem(w_ann, names_w[i], self.popvalue())
         defaultarguments = self.popvalues(posdefaults)
-        kw_defs_w = None
+        w_kw_defs = None
         if kwdefaults:
-            kw_defs_w = {}
-            for i in range(kwdefaults):
-                w_defvalue = self.popvalue()
-                w_defname = self.popvalue()
-                function.Function.add_kwdefaults(space, kw_defs_w,
-                                                 w_defname, w_defvalue)
+            w_kw_defs = space.newdict(strdict=True)
+            for i in range(kwdefaults - 1, -1, -1):
+                w_name = self.popvalue()
+                w_def = self.popvalue()
+                space.setitem(w_kw_defs, w_def, w_name)
         fn = function.Function(space, codeobj, self.get_w_globals(), defaultarguments,
-                               kw_defs_w, freevars, w_ann, qualname=qualname)
+                               w_kw_defs, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
     def MAKE_FUNCTION(self, oparg, next_instr):
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -25,13 +25,6 @@
             pass
         return -1
 
-    @jit.elidable
-    def get_kwonly_default(self, i, kw_defs_w):
-        if kw_defs_w is None:
-            raise KeyError
-        name = self.kwonlyargnames[i]
-        return kw_defs_w[name]
-
     def num_argnames(self):
         return len(self.argnames)
 

From pypy.commits at gmail.com  Sat Aug 20 02:47:23 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 19 Aug 2016 23:47:23 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: fx typo, getitem/getslice,
 some translation fixes
Message-ID: <57b7fcfb.4219c20a.c8f99.7fb4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86335:1b8ccf046b59
Date: 2016-08-20 18:46 +1200
http://bitbucket.org/pypy/pypy/changeset/1b8ccf046b59/

Log:	fx typo, getitem/getslice, some translation fixes

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -301,32 +301,29 @@
     # Similar to Py_buffer
     _immutable_ = True
 
-    def __init__(self, ptr, size, w_obj, pybuf=None):
+    def __init__(self, ptr, size, w_obj, format='B', shape=None,
+                strides=None, ndim=1, itemsize=1, readonly=True):
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
-        if pybuf is None:
-            self.format = 'B'
+        self.format = format
+        if not shape:
             self.shape = [size]
+        if not strides:
             self.strides = [1]
-            self.ndim = 1
-            self.itemsize = 1
-            self.readonly = True
-        else:
-            self.format = rffi.charp2str(pybuf.c_format)
-            self.ndim = pybuf.c_ndim
-            self.shape = [pybuf.c_shape[i] for i in range(self.ndim)]
-            self.strides = [pybuf.c_strides[i] for i in range(self.ndim)]
-            self.itemsize = pybuf.c_itemsize 
-            self.readonly = pybuf.c_readonly
+        self.ndim = ndim 
+        self.itemsize = itemsize
+        self.readonly = readonly
 
     def getlength(self):
         return self.size
 
     def getitem(self, index):
+        if self.itemsize == 1:
+            return self.ptr[index]
         start = index * self.itemsize
         stop = (index + 1) * self.itemsize
-        return ''.join([self.ptr[i] for i in range(start, stop)])
+        return self.getslice(start, stop, 1, stop - start)
 
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
@@ -337,6 +334,8 @@
     def getshape(self):
         return self.shape
 
+    def getslice(self, start, stop, step, size):
+        return ''.join([self.ptr[i] for i in range(start, stop, step)])
 
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
@@ -358,7 +357,14 @@
             space.fromcache(State).check_and_raise_exception(always=True)
         ptr = pybuf.c_buf
         size = pybuf.c_len
-        return space.newbuffer(CPyBuffer(ptr, size, w_self, pybuf))
+        ndim = pybuf.c_ndim
+        shape =   [pybuf.c_shape[i]   for i in range(ndim)]
+        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        format = rffi.charp2str(pybuf.c_format)
+        return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
+                            ndim=ndim, shape=shape, strides=strides,
+                            itemsize=pybuf.c_itemsize,
+                            readonly=pybuf.c_readonly))
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
@@ -921,9 +927,9 @@
         return 2
     if slotdef.slot_name.startswith('tp_as_sequence'):
         return 3
-    if slodef.slot_name == 'tp_as_buffer.c_bf_getbuffer':
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getbuffer':
         return 100
-    if slodef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer':
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer':
         return 101
     return 0
 slotdefs = sorted(slotdefs, key=slotdef_sort_key)
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -13,7 +13,7 @@
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
         w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
-        assert w_char == space.wrap('h')
+        assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 

From pypy.commits at gmail.com  Sat Aug 20 03:18:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 00:18:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: A much simpler solution
 gets us most of the way there (thanks fijal):
Message-ID: <57b80441.43681c0a.d0d8.a3c4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86336:d428c0d4802a
Date: 2016-08-20 09:15 +0200
http://bitbucket.org/pypy/pypy/changeset/d428c0d4802a/

Log:	A much simpler solution gets us most of the way there (thanks
	fijal): rely on the mapdict logic

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -38,7 +38,9 @@
                           'name?',
                           'w_kw_defs?']
 
-    def __init__(self, space, code, w_globals=None, defs_w=[], w_kw_defs=None,
+    w_kw_defs = None
+
+    def __init__(self, space, code, w_globals=None, defs_w=[], kw_defs_w=None,
                  closure=None, w_ann=None, forcename=None, qualname=None):
         self.space = space
         self.name = forcename or code.co_name
@@ -48,10 +50,12 @@
         self.w_func_globals = w_globals  # the globals dictionary
         self.closure = closure    # normally, list of Cell instances or None
         self.defs_w = defs_w
-        self.w_kw_defs = w_kw_defs
         self.w_func_dict = None # filled out below if needed
         self.w_module = None
         self.w_ann = w_ann
+        #
+        if kw_defs_w is not None:
+            self.init_kwdefaults_dict(kw_defs_w)
 
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
@@ -379,14 +383,29 @@
 
     def fset_func_kwdefaults(self, space, w_new):
         if space.is_w(w_new, space.w_None):
-            w_new = None
-        elif not space.isinstance_w(w_new, space.w_dict):
-            raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
-        self.w_kw_defs = w_new
+            self.w_kw_defs = None
+        else:
+            if not space.isinstance_w(w_new, space.w_dict):
+                raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
+            w_instance = self.init_kwdefaults_dict()
+            w_instance.setdict(space, w_new)
+            self.w_kw_defs = w_instance.getdict(space)
 
     def fdel_func_kwdefaults(self, space):
         self.w_kw_defs = None
 
+    def init_kwdefaults_dict(self, kw_defs_w=[]):
+        # use the mapdict logic to get at least not-too-bad JIT code
+        # from function calls with default values of kwonly arguments
+        space = self.space
+        w_class = space.fromcache(KwDefsClassCache).w_class
+        w_instance = space.call_function(w_class)
+        for w_name, w_value in kw_defs_w:
+            attr = space.unicode_w(w_name).encode('utf-8')
+            w_instance.setdictvalue(space, attr, w_value)
+        self.w_kw_defs = w_instance.getdict(space)
+        return w_instance
+
     def fget_func_doc(self, space):
         if self.w_doc is None:
             self.w_doc = self.code.getdocstring(space)
@@ -663,11 +682,12 @@
     def __init__(self, func):
         assert isinstance(func, Function)
         Function.__init__(self, func.space, func.code, func.w_func_globals,
-                          func.defs_w, func.w_kw_defs, func.closure,
+                          func.defs_w, None, func.closure,
                           None, func.name)
         self.w_doc = func.w_doc
         self.w_func_dict = func.w_func_dict
         self.w_module = func.w_module
+        self.w_kw_defs = func.w_kw_defs
 
     def descr_builtinfunction__new__(space, w_subtype):
         raise oefmt(space.w_TypeError,
@@ -685,3 +705,12 @@
     else:
         code = None
     return isinstance(code, BuiltinCode)
+
+
+class KwDefsClassCache:
+    def __init__(self, space):
+        self.w_class = space.appexec([], """():
+            class KwDefs:
+                pass
+            return KwDefs
+        """)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1235,15 +1235,15 @@
             for i in range(len(names_w) - 1, -1, -1):
                 space.setitem(w_ann, names_w[i], self.popvalue())
         defaultarguments = self.popvalues(posdefaults)
-        w_kw_defs = None
+        kw_defs_w = None
         if kwdefaults:
-            w_kw_defs = space.newdict(strdict=True)
-            for i in range(kwdefaults - 1, -1, -1):
-                w_name = self.popvalue()
-                w_def = self.popvalue()
-                space.setitem(w_kw_defs, w_def, w_name)
+            kw_defs_w = []
+            for i in range(kwdefaults):
+                w_defvalue = self.popvalue()
+                w_defname = self.popvalue()
+                kw_defs_w.append((w_defname, w_defvalue))
         fn = function.Function(space, codeobj, self.get_w_globals(), defaultarguments,
-                               w_kw_defs, freevars, w_ann, qualname=qualname)
+                               kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
     def MAKE_FUNCTION(self, oparg, next_instr):

From pypy.commits at gmail.com  Sat Aug 20 03:47:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 00:47:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Remove the "from __future__ import xyz"
 from the app-level helpers: it
Message-ID: <57b80b01.031dc20a.34dd5.99db@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86337:99c145db7090
Date: 2016-08-20 09:39 +0200
http://bitbucket.org/pypy/pypy/changeset/99c145db7090/

Log:	Remove the "from __future__ import xyz" from the app-level helpers:
	it has no effect any more, and it loads the world by using importlib

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -1167,15 +1167,15 @@
             source = source[source.find('\n') + 1:].lstrip()
         assert source.startswith("def "), "can only transform functions"
         source = source[4:]
-        import __future__
-        if flags & __future__.CO_FUTURE_DIVISION:
-            prefix += "from __future__ import division\n"
-        if flags & __future__.CO_FUTURE_ABSOLUTE_IMPORT:
-            prefix += "from __future__ import absolute_import\n"
-        if flags & __future__.CO_FUTURE_PRINT_FUNCTION:
-            prefix += "from __future__ import print_function\n"
-        if flags & __future__.CO_FUTURE_UNICODE_LITERALS:
-            prefix += "from __future__ import unicode_literals\n"
+        # The following flags have no effect any more in app-level code
+        # (i.e. they are always on anyway), and have been removed:
+        #    CO_FUTURE_DIVISION
+        #    CO_FUTURE_ABSOLUTE_IMPORT
+        #    CO_FUTURE_PRINT_FUNCTION
+        #    CO_FUTURE_UNICODE_LITERALS
+        # Original code was, for each of these flags:
+        #    if flags & __future__.CO_xxx:
+        #        prefix += "from __future__ import yyy\n"
     p = source.find('(')
     assert p >= 0
     funcname = source[:p].strip()

From pypy.commits at gmail.com  Sat Aug 20 03:47:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 00:47:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: hg merge py3k
Message-ID: <57b80b02.56421c0a.ef4c7.ae2a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86338:533d2eb4b9de
Date: 2016-08-20 09:40 +0200
http://bitbucket.org/pypy/pypy/changeset/533d2eb4b9de/

Log:	hg merge py3k

diff --git a/lib-python/3/test/test_compileall.py b/lib-python/3/test/test_compileall.py
--- a/lib-python/3/test/test_compileall.py
+++ b/lib-python/3/test/test_compileall.py
@@ -202,11 +202,10 @@
 
     # Ensure that the default behavior of compileall's CLI is to create
     # PEP 3147 pyc/pyo files.
-    _pyo = 'pyo' if support.check_impl_detail() else 'pyc'
     for name, ext, switch in [
         ('normal', 'pyc', []),
-        ('optimize', _pyo, ['-O']),
-        ('doubleoptimize', _pyo, ['-OO']),
+        ('optimize', 'pyo', ['-O']),
+        ('doubleoptimize', 'pyo', ['-OO']),
     ]:
         def f(self, ext=ext, switch=switch):
             script_helper.assert_python_ok(*(switch +
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -1190,15 +1190,15 @@
             source = source[source.find('\n') + 1:].lstrip()
         assert source.startswith("def "), "can only transform functions"
         source = source[4:]
-        import __future__
-        if flags & __future__.CO_FUTURE_DIVISION:
-            prefix += "from __future__ import division\n"
-        if flags & __future__.CO_FUTURE_ABSOLUTE_IMPORT:
-            prefix += "from __future__ import absolute_import\n"
-        if flags & __future__.CO_FUTURE_PRINT_FUNCTION:
-            prefix += "from __future__ import print_function\n"
-        if flags & __future__.CO_FUTURE_UNICODE_LITERALS:
-            prefix += "from __future__ import unicode_literals\n"
+        # The following flags have no effect any more in app-level code
+        # (i.e. they are always on anyway), and have been removed:
+        #    CO_FUTURE_DIVISION
+        #    CO_FUTURE_ABSOLUTE_IMPORT
+        #    CO_FUTURE_PRINT_FUNCTION
+        #    CO_FUTURE_UNICODE_LITERALS
+        # Original code was, for each of these flags:
+        #    if flags & __future__.CO_xxx:
+        #        prefix += "from __future__ import yyy\n"
     p = source.find('(')
     assert p >= 0
     funcname = source[:p].strip()
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -327,7 +327,8 @@
             i += 1
             if ch == '"':
                 content_utf8 = builder.build()
-                content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
+                content_unicode = unicodehelper.decode_utf8(
+                    self.space, content_utf8, allow_surrogates=True)
                 self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.wrap(content_unicode)
@@ -374,7 +375,8 @@
                    # this point
         #
         uchr = runicode.code_to_unichr(val)     # may be a surrogate pair again
-        utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
+        utf8_ch = unicodehelper.encode_utf8(
+            self.space, uchr, allow_surrogates=True)
         builder.append(utf8_ch)
         return i
 
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -10,10 +10,10 @@
     assert dec.skip_whitespace(8) == len(s)
     dec.close()
 
-    
+
 
 class AppTest(object):
-    spaceconfig = {"objspace.usemodules._pypyjson": True}
+    spaceconfig = {"usemodules": ['_pypyjson']}
 
     def test_raise_on_bytes(self):
         import _pypyjson
@@ -40,7 +40,7 @@
         raises(ValueError, _pypyjson.loads, 'fa')
         raises(ValueError, _pypyjson.loads, 'f')
         raises(ValueError, _pypyjson.loads, 'falXX')
-        
+
 
     def test_decode_string(self):
         import _pypyjson
@@ -69,7 +69,7 @@
         import _pypyjson
         assert _pypyjson.loads(r'"\\"') == '\\'
         assert _pypyjson.loads(r'"\""') == '"'
-        assert _pypyjson.loads(r'"\/"') == '/'       
+        assert _pypyjson.loads(r'"\/"') == '/'
         assert _pypyjson.loads(r'"\b"') == '\b'
         assert _pypyjson.loads(r'"\f"') == '\f'
         assert _pypyjson.loads(r'"\n"') == '\n'
@@ -85,7 +85,7 @@
         import _pypyjson
         s = r'"hello\nworld' # missing the trailing "
         raises(ValueError, "_pypyjson.loads(s)")
-        
+
     def test_escape_sequence_unicode(self):
         import _pypyjson
         s = r'"\u1234"'
@@ -166,7 +166,7 @@
     def test_decode_object_nonstring_key(self):
         import _pypyjson
         raises(ValueError, "_pypyjson.loads('{42: 43}')")
-        
+
     def test_decode_array(self):
         import _pypyjson
         assert _pypyjson.loads('[]') == []
@@ -183,7 +183,7 @@
         res = _pypyjson.loads('"z\\ud834\\udd20x"')
         assert res == expected
 
-    def test_surrogate_pair(self):
+    def test_lone_surrogate(self):
         import _pypyjson
         json = '{"a":"\\uD83D"}'
         res = _pypyjson.loads(json)

From pypy.commits at gmail.com  Sat Aug 20 05:23:45 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 20 Aug 2016 02:23:45 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: move itemsize logic to
 W_MemoryView (arigato), set valid defaults in Buffer, typos
Message-ID: <57b821a1.c186c20a.63b65.afb6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86339:c7b6199d9a55
Date: 2016-08-20 21:22 +1200
http://bitbucket.org/pypy/pypy/changeset/c7b6199d9a55/

Log:	move itemsize logic to W_MemoryView (arigato), set valid defaults in
	Buffer, typos

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -319,11 +319,7 @@
         return self.size
 
     def getitem(self, index):
-        if self.itemsize == 1:
-            return self.ptr[index]
-        start = index * self.itemsize
-        stop = (index + 1) * self.itemsize
-        return self.getslice(start, stop, 1, stop - start)
+        return self.ptr[index]
 
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
@@ -334,8 +330,8 @@
     def getshape(self):
         return self.shape
 
-    def getslice(self, start, stop, step, size):
-        return ''.join([self.ptr[i] for i in range(start, stop, step)])
+    def getitemsize(self):
+        return self.itemsize
 
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
@@ -586,8 +582,8 @@
             return space.call_args(space.get(new_fn, w_self), args)
         api_func = slot_tp_new.api_func
     elif name == 'tp_as_buffer.c_bf_getbuffer':
-        buf_fn = w_type.getdictvalue(space, '__buffer__')
-        if buf_fn is None:
+        buff_fn = w_type.getdictvalue(space, '__buffer__')
+        if buff_fn is None:
             return
         @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
                 rffi.INT_real, header=None, error=-1)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -73,6 +73,15 @@
 
     def descr_getitem(self, space, w_index):
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        itemsize = self.buf.getitemsize()
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+            stop  = start + size
+            if step == 0:
+                step = 1
+            if stop > self.getlength():
+                raise oefmt(space.w_IndexError, 'index out of range')
         if step not in (0, 1):
             raise oefmt(space.w_NotImplementedError, "")
         if step == 0:  # index only
@@ -85,6 +94,15 @@
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        itemsize = self.buf.getitemsize()
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+            stop  = start + size
+            if step == 0:
+                step = 1
+            if stop > self.getlength():
+                raise oefmt(space.w_IndexError, 'index out of range')
         if step not in (0, 1):
             raise oefmt(space.w_NotImplementedError, "")
         value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -60,13 +60,13 @@
         raise ValueError("no raw buffer")
 
     def getformat(self):
-        raise NotImplementedError
+        return 'B'
 
     def getitemsize(self):
-        raise NotImplementedError
+        return 1
 
     def getndim(self):
-        raise NotImplementedError
+        return 1
 
     def getshape(self):
         return [self.getlength()]

From pypy.commits at gmail.com  Sat Aug 20 06:05:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 03:05:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix for test_close_on_collect
Message-ID: <57b82b82.c62f1c0a.9f43d.de95@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86340:d5c8c1282a7d
Date: 2016-08-20 12:05 +0200
http://bitbucket.org/pypy/pypy/changeset/d5c8c1282a7d/

Log:	Fix for test_close_on_collect

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,7 +1,7 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
-from rpython.rlib import jit
+from rpython.rlib import jit, rweakref
 from rpython.rlib.debug import make_sure_not_resized, check_nonneg
 from rpython.rlib.jit import hint
 from rpython.rlib.objectmodel import instantiate, specialize, we_are_translated
@@ -62,7 +62,8 @@
     __metaclass__ = extendabletype
 
     frame_finished_execution = False
-    frame_generator          = None    # for generators/coroutines
+    f_generator_wref         = rweakref.dead_ref  # for generators/coroutines
+    f_generator_nowref       = None               # (only one of the two attrs)
     last_instr               = -1
     last_exception           = None
     f_backref                = jit.vref_None
@@ -239,17 +240,23 @@
             self.locals_cells_stack_w[index] = outer_func.closure[i]
             index += 1
 
+    def _is_generator_or_coroutine(self):
+        return (self.getcode().co_flags & (pycode.CO_COROUTINE |
+                                           pycode.CO_GENERATOR)) != 0
+
     def run(self):
         """Start this frame's execution."""
-        if self.getcode().co_flags & (pycode.CO_COROUTINE |
-                                      pycode.CO_GENERATOR):
+        if self._is_generator_or_coroutine():
             if self.getcode().co_flags & pycode.CO_COROUTINE:
                 from pypy.interpreter.generator import Coroutine
                 gen = Coroutine(self)
             else:
                 from pypy.interpreter.generator import GeneratorIterator
                 gen = GeneratorIterator(self)
-            self.frame_generator = gen
+            if self.space.config.translation.rweakref:
+                self.f_generator_wref = rweakref.ref(gen)
+            else:
+                self.f_generator_nowref = gen
             return self.space.wrap(gen)
         else:
             return self.execute_frame()
@@ -897,12 +904,20 @@
         # is not quite like CPython when it clears f_exc_* (however
         # there might not be an observable difference).
         if not self.frame_finished_execution:
-            if self.frame_generator is None or self.frame_generator.running:
+            if not self._is_generator_or_coroutine():
                 raise oefmt(space.w_RuntimeError,
                             "cannot clear an executing frame")
-            # xxx CPython raises the RuntimeWarning "coroutine was never
-            # awaited" in this case too.  Does it make any sense?
-            self.frame_generator.descr_close()
+            if space.config.translation.rweakref:
+                gen = self.f_generator_wref()
+            else:
+                gen = self.f_generator_nowref
+            if gen is not None:
+                if gen.running:
+                    raise oefmt(space.w_RuntimeError,
+                                "cannot clear an executing frame")
+                # xxx CPython raises the RuntimeWarning "coroutine was never
+                # awaited" in this case too.  Does it make any sense?
+                gen.descr_close()
 
         self.last_exception = None
         debug = self.getdebug()
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -220,22 +220,17 @@
         raises(RuntimeError, g.close)
 
     def test_close_on_collect(self):
-        ## we need to exec it, else it won't run on python2.4
-        d = {}
-        exec("""
         def f():
             try:
                 yield
             finally:
                 f.x = 42
-        """.strip(), d, d)
-
-        g = d['f']()
+        g = f()
         next(g)
         del g
         import gc
         gc.collect()
-        assert d['f'].x == 42
+        assert f.x == 42
 
     def test_generator_raises_typeerror(self):
         def f():

From pypy.commits at gmail.com  Sat Aug 20 06:15:34 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 20 Aug 2016 03:15:34 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: progress with translation,
 trying to raise clear exceptions rahter than segfault
Message-ID: <57b82dc6.54bc1c0a.66935.ea91@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86341:c99ca063d648
Date: 2016-08-20 22:14 +1200
http://bitbucket.org/pypy/pypy/changeset/c99ca063d648/

Log:	progress with translation, trying to raise clear exceptions rahter
	than segfault

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -12,8 +12,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    import pdb; pdb.set_trace()
-    raise NotImplementedError
+    raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
 def PyMemoryView_GET_BUFFER(space, w_obj):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -3,6 +3,7 @@
 import re
 
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
     mangle_name, pypy_decl, Py_buffer, Py_bufferP)
@@ -344,7 +345,6 @@
 
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
-    # XXX leak
     with lltype.scoped_alloc(Py_buffer) as pybuf:
         # XXX flags are not in w_args?
         flags = rffi.cast(rffi.INT_real,0)
@@ -353,14 +353,14 @@
             space.fromcache(State).check_and_raise_exception(always=True)
         ptr = pybuf.c_buf
         size = pybuf.c_len
-        ndim = pybuf.c_ndim
+        ndim = widen(pybuf.c_ndim)
         shape =   [pybuf.c_shape[i]   for i in range(ndim)]
         strides = [pybuf.c_strides[i] for i in range(ndim)]
         format = rffi.charp2str(pybuf.c_format)
         return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
                             ndim=ndim, shape=shape, strides=strides,
                             itemsize=pybuf.c_itemsize,
-                            readonly=pybuf.c_readonly))
+                            readonly=widen(pybuf.c_readonly)))
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
@@ -588,11 +588,13 @@
         @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
                 rffi.INT_real, header=None, error=-1)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
-        def buff_w(space, w_self, w_args, w_kwds):
+        def buff_w(space, w_self, pybuf, flags):
             # XXX this is wrong, needs a test
-            args = Arguments(space, [w_self],
-                             w_stararg=w_args, w_starstararg=w_kwds)
-            return space.call_args(space.get(buff_fn, w_self), args)
+            raise oefmt(space.w_NotImplemented, 
+                "calling bf_getbuffer on a builtin type not supported yet")
+            #args = Arguments(space, [w_self],
+            #                 w_stararg=w_args, w_starstararg=w_kwds)
+            #return space.call_args(space.get(buff_fn, w_self), args)
         api_func = buff_w.api_func
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce

From pypy.commits at gmail.com  Sat Aug 20 06:34:37 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 20 Aug 2016 03:34:37 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: translation fix
Message-ID: <57b8323d.28eac20a.e7e57.d078@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86342:33ec1bbbd5b9
Date: 2016-08-20 22:33 +1200
http://bitbucket.org/pypy/pypy/changeset/33ec1bbbd5b9/

Log:	translation fix

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -349,7 +349,7 @@
         # XXX flags are not in w_args?
         flags = rffi.cast(rffi.INT_real,0)
         size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
-        if size < 0:
+        if widen(size) < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
         ptr = pybuf.c_buf
         size = pybuf.c_len

From pypy.commits at gmail.com  Sat Aug 20 09:46:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 06:46:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: in-progress
Message-ID: <57b85f50.6974c20a.af6a3.16f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86343:0cb8095f17f5
Date: 2016-08-20 15:46 +0200
http://bitbucket.org/pypy/pypy/changeset/0cb8095f17f5/

Log:	in-progress

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -28,6 +28,8 @@
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
 
+NO_DEFAULT = object()
+
 
 # internal non-translatable parts:
 class SignatureBuilder(object):
@@ -967,68 +969,63 @@
         self.name = app_name
         self.as_classmethod = as_classmethod
 
-        if not f.func_defaults:
-            self._staticdefs = []
-        else:
-            argnames = self._code._argnames
-            defaults = f.func_defaults
-            self._staticdefs = zip(argnames[-len(defaults):], defaults)
+        argnames = self._code._argnames
+        defaults = f.func_defaults or ()
+        self._staticdefs = dict(zip(
+            argnames[len(argnames) - len(defaults):], defaults))
+
         return self
 
     def _getdefaults(self, space):
         "NOT_RPYTHON"
-        defs_w = []
-        if self._code.sig.kwonlyargnames:
-            import pdb; pdb.set_trace()
-        unwrap_spec = self._code._unwrap_spec[-len(self._staticdefs):]
-        for i, (name, defaultval) in enumerate(self._staticdefs):
+        alldefs_w = {}
+        assert len(self._code._argnames) == len(self._code._unwrap_spec)
+        for name, spec in zip(self._code._argnames, self._code._unwrap_spec):
+            if name == '__kwonly__':
+                continue
+
+            defaultval = self._staticdefs.get(name, NO_DEFAULT)
             if name.startswith('w_'):
-                assert defaultval is None, (
+                assert defaultval in (NO_DEFAULT, None), (
                     "%s: default value for '%s' can only be None, got %r; "
                     "use unwrap_spec(...=WrappedDefault(default))" % (
                     self._code.identifier, name, defaultval))
-                defs_w.append(None)
-            elif name != '__args__' and name != 'args_w':
-                spec = unwrap_spec[i]
-                if isinstance(defaultval, str) and spec not in [str]:
-                    defs_w.append(space.newbytes(defaultval))
-                else:
-                    defs_w.append(space.wrap(defaultval))
-        if self._code._unwrap_spec:
-            UNDEFINED = object()
-            allargnames = (self._code.sig.argnames +
-                           self._code.sig.kwonlyargnames)
-            alldefs_w = [UNDEFINED] * len(allargnames)
-            if defs_w:
-                alldefs_w[-len(defs_w):] = defs_w
-            code = self._code
-            assert isinstance(code._unwrap_spec, (list, tuple))
-            assert isinstance(code._argnames, list)
-            assert len(code._unwrap_spec) == len(code._argnames)
-            for i in range(len(code._unwrap_spec)-1, -1, -1):
-                spec = code._unwrap_spec[i]
-                argname = code._argnames[i]
-                if isinstance(spec, tuple) and spec[0] is W_Root:
-                    assert False, "use WrappedDefault"
-                if isinstance(spec, WrappedDefault):
-                    default_value = spec.default_value
-                    if isinstance(default_value, str):
-                        w_default = space.newbytes(default_value)
+
+            if isinstance(spec, tuple) and spec[0] is W_Root:
+                assert False, "use WrappedDefault"
+            elif isinstance(spec, WrappedDefault):
+                assert name.startswith('w_')
+                defaultval = spec.default_value
+
+            if defaultval is not NO_DEFAULT:
+                if name != '__args__' and name != 'args_w':
+                    if isinstance(defaultval, str) and spec not in [str]:
+                        w_def = space.newbytes(defaultval)
                     else:
-                        w_default = space.wrap(default_value)
-                    assert isinstance(w_default, W_Root)
-                    assert argname.startswith('w_')
-                    argname = argname[2:]
-                    j = allargnames.index(argname)
-                    assert alldefs_w[j] in (UNDEFINED, None)
-                    alldefs_w[j] = w_default
-            first_defined = 0
-            while (first_defined < len(alldefs_w) and
-                   alldefs_w[first_defined] is UNDEFINED):
-                first_defined += 1
-            defs_w = alldefs_w[first_defined:]
-            assert UNDEFINED not in defs_w
-        return defs_w
+                        w_def = space.wrap(defaultval)
+                    alldefs_w[name] = w_def
+        #
+        # Here, 'alldefs_w' maps some argnames to their wrapped default
+        # value.  We return two lists:
+        #  - a list of defaults for positional arguments, which covers
+        #    some suffix of the sig.argnames list
+        #  - a list of pairs (w_name, w_def) for kwonly arguments
+        #
+        sig = self._code.sig
+        first_defined = 0
+        while (first_defined < len(sig.argnames) and
+               sig.argnames[first_defined] not in alldefs_w):
+            first_defined += 1
+        defs_w = [alldefs_w.pop(name) for name in sig.argnames[first_defined:]]
+
+        kw_defs_w = None
+        if alldefs_w:
+            kw_defs_w = []
+            for name, w_def in sorted(alldefs_w.items()):
+                w_name = space.newunicode(name.decode('utf-8'))
+                kw_defs_w.append((w_name, w_def))
+
+        return defs_w, kw_defs_w
 
     # lazy binding to space
 
@@ -1048,10 +1045,9 @@
     def build(cache, gateway):
         "NOT_RPYTHON"
         space = cache.space
-        defs = gateway._getdefaults(space)
+        defs_w, kw_defs_w = gateway._getdefaults(space)
         code = gateway._code
-        w_kw_defs = None #XXXXXXXXXXXXXXXXXXX
-        fn = FunctionWithFixedCode(space, code, None, defs, w_kw_defs,
+        fn = FunctionWithFixedCode(space, code, None, defs_w, kw_defs_w,
                                    forcename=gateway.name)
         if not space.config.translating:
             fn.add_to_table()

From pypy.commits at gmail.com  Sat Aug 20 09:55:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 06:55:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: fixes
Message-ID: <57b86139.a111c20a.15d15.180f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86344:90e41bfacc6b
Date: 2016-08-20 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/90e41bfacc6b/

Log:	fixes

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -985,11 +985,14 @@
                 continue
 
             defaultval = self._staticdefs.get(name, NO_DEFAULT)
+            w_def = Ellipsis
             if name.startswith('w_'):
                 assert defaultval in (NO_DEFAULT, None), (
                     "%s: default value for '%s' can only be None, got %r; "
                     "use unwrap_spec(...=WrappedDefault(default))" % (
                     self._code.identifier, name, defaultval))
+                if defaultval is None:
+                    w_def = None
 
             if isinstance(spec, tuple) and spec[0] is W_Root:
                 assert False, "use WrappedDefault"
@@ -999,10 +1002,13 @@
 
             if defaultval is not NO_DEFAULT:
                 if name != '__args__' and name != 'args_w':
-                    if isinstance(defaultval, str) and spec not in [str]:
-                        w_def = space.newbytes(defaultval)
-                    else:
-                        w_def = space.wrap(defaultval)
+                    if w_def is Ellipsis:
+                        if isinstance(defaultval, str) and spec not in [str]:
+                            w_def = space.newbytes(defaultval)
+                        else:
+                            w_def = space.wrap(defaultval)
+                    if name.startswith('w_'):
+                        name = name[2:]
                     alldefs_w[name] = w_def
         #
         # Here, 'alldefs_w' maps some argnames to their wrapped default
@@ -1022,6 +1028,7 @@
         if alldefs_w:
             kw_defs_w = []
             for name, w_def in sorted(alldefs_w.items()):
+                assert name in sig.kwonlyargnames
                 w_name = space.newunicode(name.decode('utf-8'))
                 kw_defs_w.append((w_name, w_def))
 
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -759,7 +759,7 @@
         @gateway.unwrap_spec(w_x = WrappedDefault(42), y=int)
         def g(space, w_x, y):
             never_called
-        py.test.raises(AssertionError, space.wrap, gateway.interp2app_temp(g))
+        py.test.raises(KeyError, space.wrap, gateway.interp2app_temp(g))
 
     def test_unwrap_spec_default_applevel_bug2(self):
         space = self.space

From pypy.commits at gmail.com  Sat Aug 20 10:00:42 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sat, 20 Aug 2016 07:00:42 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Fix startup code (again?!)
Message-ID: <57b8628a.08d11c0a.c6951.3ce6@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86345:e2f94675d8b2
Date: 2016-08-20 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/e2f94675d8b2/

Log:	Fix startup code (again?!)

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -472,10 +472,10 @@
         return eci
 
     def gc_startup_code(self):
+        yield 'qcgc_initialize();'
         s = list(super(QcgcFrameworkGcPolicy, self).gc_startup_code())
         for i in s:
             yield i
-        yield 'qcgc_initialize();'
 
 name_to_gcpolicy = {
     'boehm': BoehmGcPolicy,

From pypy.commits at gmail.com  Sat Aug 20 10:31:39 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 20 Aug 2016 07:31:39 -0700 (PDT)
Subject: [pypy-commit] pypy const-fold-we-are-jitted: add the corresponding
 code in all.py
Message-ID: <57b869cb.c19d1c0a.a7ff3.472e@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: const-fold-we-are-jitted
Changeset: r86347:02fc2f32a831
Date: 2016-08-20 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/02fc2f32a831/

Log:	add the corresponding code in all.py

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -261,6 +261,9 @@
                    "stack based virtual machines (only for backends that support it)",
                    default=True),
         BoolOption("storesink", "Perform store sinking", default=True),
+        BoolOption("replace_we_are_jitted",
+                   "Replace we_are_jitted() calls by False",
+                   default=False, cmdline=None),
         BoolOption("none",
                    "Do not run any backend optimizations",
                    requires=[('translation.backendopt.inline', False),
diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py
--- a/rpython/translator/backendopt/all.py
+++ b/rpython/translator/backendopt/all.py
@@ -2,6 +2,7 @@
 from rpython.translator.backendopt import inline
 from rpython.translator.backendopt.malloc import remove_mallocs
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.translator.backendopt.stat import print_statistics
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator import simplify
@@ -36,6 +37,7 @@
     # inline_threshold, mallocs
     # merge_if_blocks, constfold, heap2stack
     # clever_malloc_removal, remove_asserts
+    # replace_we_are_jitted
 
     config = translator.config.translation.backendopt.copy(as_default=True)
     config.set(**kwds)
@@ -49,6 +51,10 @@
         print "before optimizations:"
         print_statistics(translator.graphs[0], translator, "per-graph.txt")
 
+    if config.replace_we_are_jitted:
+        for graph in graphs:
+            replace_we_are_jitted(graph)
+
     if config.remove_asserts:
         constfold(config, graphs)
         remove_asserts(translator, graphs)
diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py
--- a/rpython/translator/backendopt/test/test_all.py
+++ b/rpython/translator/backendopt/test/test_all.py
@@ -289,3 +289,19 @@
         llinterp = LLInterpreter(t.rtyper)
         res = llinterp.eval_graph(later_graph, [10])
         assert res == 1
+
+    def test_replace_we_are_jitted(self):
+        from rpython.rlib import jit
+        def f():
+            if jit.we_are_jitted():
+                return 1
+            return 2 + jit.we_are_jitted()
+
+        t = self.translateopt(f, [])
+        graph = graphof(t, f)
+        # by default, replace_we_are_jitted is off
+        assert graph.startblock.operations[0].args[0].value is jit._we_are_jitted
+
+        t = self.translateopt(f, [], replace_we_are_jitted=True)
+        graph = graphof(t, f)
+        assert graph.startblock.exits[0].args[0].value == 2
diff --git a/rpython/translator/backendopt/test/test_constfold.py b/rpython/translator/backendopt/test/test_constfold.py
--- a/rpython/translator/backendopt/test/test_constfold.py
+++ b/rpython/translator/backendopt/test/test_constfold.py
@@ -346,8 +346,6 @@
     check_graph(graph, [], 66, t)
 
 def test_replace_we_are_jitted():
-    from rpython.flowspace import model
-    from rpython.rtyper.lltypesystem import lltype
     from rpython.rlib import jit
     def fn():
         if jit.we_are_jitted():

From pypy.commits at gmail.com  Sat Aug 20 10:31:41 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 20 Aug 2016 07:31:41 -0700 (PDT)
Subject: [pypy-commit] pypy const-fold-we-are-jitted: replace we_are_jitted
 with False when doing the normal backendopt optimization, 
Message-ID: <57b869cd.81a2c20a.9de76.26b1@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: const-fold-we-are-jitted
Changeset: r86348:b2dbc4e9bdea
Date: 2016-08-20 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/b2dbc4e9bdea/

Log:	replace we_are_jitted with False when doing the normal backendopt
	optimization, but leave it alone when running the backend
	optimizations from the JIT

diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -452,7 +452,8 @@
                               merge_if_blocks=True,
                               constfold=True,
                               remove_asserts=True,
-                              really_remove_asserts=True)
+                              really_remove_asserts=True,
+                              replace_we_are_jitted=False)
 
     def prejit_optimizations_minimal_inline(self, policy, graphs):
         from rpython.translator.backendopt.inline import auto_inline_graphs
diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -381,7 +381,7 @@
         """ Run all backend optimizations - lltype version
         """
         from rpython.translator.backendopt.all import backend_optimizations
-        backend_optimizations(self.translator)
+        backend_optimizations(self.translator, replace_we_are_jitted=True)
 
 
     STACKCHECKINSERTION = 'stackcheckinsertion_lltype'
diff --git a/rpython/translator/test/test_interactive.py b/rpython/translator/test/test_interactive.py
--- a/rpython/translator/test/test_interactive.py
+++ b/rpython/translator/test/test_interactive.py
@@ -78,3 +78,15 @@
     dll = ctypes.CDLL(str(t.driver.c_entryp))
     f = dll.pypy_g_f
     assert f(2, 3) == 5
+
+def test_check_that_driver_uses_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def f():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+
+    t = Translation(f, [])
+    t.backendopt()
+    graph = t.driver.translator.graphs[0]
+    assert graph.startblock.exits[0].args[0].value == 2

From pypy.commits at gmail.com  Sat Aug 20 10:31:36 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 20 Aug 2016 07:31:36 -0700 (PDT)
Subject: [pypy-commit] pypy const-fold-we-are-jitted: an optimization to
 replace we_are_jitted() with 0
Message-ID: <57b869c8.8a13c20a.86f4a.2997@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: const-fold-we-are-jitted
Changeset: r86346:aabd56db32d1
Date: 2016-08-20 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/aabd56db32d1/

Log:	an optimization to replace we_are_jitted() with 0

diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py
--- a/rpython/translator/backendopt/constfold.py
+++ b/rpython/translator/backendopt/constfold.py
@@ -276,3 +276,25 @@
             rewire_links(splitblocks, graph)
         if not diffused and not splitblocks:
             break # finished
+
+def replace_symbolic(graph, symbolic, value):
+    result = False
+    for block in graph.iterblocks():
+        for op in block.operations:
+            for i, arg in enumerate(op.args):
+                if isinstance(arg, Constant) and arg.value is symbolic:
+                    op.args[i] = value
+                    result = True
+        if block.exitswitch is symbolic:
+            block.exitswitch = value
+            result = True
+    return result
+
+def replace_we_are_jitted(graph):
+    from rpython.rlib import jit
+    replacement = Constant(0)
+    replacement.concretetype = lltype.Signed
+    did_replacement = replace_symbolic(graph, jit._we_are_jitted, replacement)
+    if did_replacement:
+        constant_fold_graph(graph)
+    return did_replacement
diff --git a/rpython/translator/backendopt/test/test_constfold.py b/rpython/translator/backendopt/test/test_constfold.py
--- a/rpython/translator/backendopt/test/test_constfold.py
+++ b/rpython/translator/backendopt/test/test_constfold.py
@@ -7,6 +7,7 @@
 from rpython.rtyper import rclass
 from rpython.rlib import objectmodel
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.conftest import option
 
 def get_graph(fn, signature):
@@ -343,3 +344,20 @@
     merge_if_blocks.merge_if_blocks_once(graph)
     constant_fold_graph(graph)
     check_graph(graph, [], 66, t)
+
+def test_replace_we_are_jitted():
+    from rpython.flowspace import model
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rlib import jit
+    def fn():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+    graph, t = get_graph(fn, [])
+    result = replace_we_are_jitted(graph)
+    assert result
+    checkgraph(graph)
+    # check shape of graph
+    assert len(graph.startblock.operations) == 0
+    assert graph.startblock.exitswitch is None
+    assert graph.startblock.exits[0].target.exits[0].args[0].value == 2

From pypy.commits at gmail.com  Sat Aug 20 11:41:54 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sat, 20 Aug 2016 08:41:54 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Try implementing allocation
Message-ID: <57b87a42.eeb8c20a.7e972.4137@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86349:3c501a577097
Date: 2016-08-20 17:03 +0200
http://bitbucket.org/pypy/pypy/changeset/3c501a577097/

Log:	Try implementing allocation

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -23,11 +23,12 @@
             ('hash', lltype.Signed))
     #HDR = rffi.COpaque('object_t')
 
-    def malloc_fixedsize_clear(self, typeid16, size,
+    def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False,
                                is_finalizer_light=False,
                                contains_weakptr=False):
-        raise NotImplementedError
+        # What is the llmemory.GCREF for?
+        return llop.qcgc_allocate(llmemory.GCREF, size, typeid)
         ## XXX finalizers are ignored for now
         ##ll_assert(not needs_finalizer, 'XXX needs_finalizer')
         ##ll_assert(not is_finalizer_light, 'XXX is_finalizer_light')
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -514,6 +514,9 @@
     # NOTE NOTE NOTE! don't forget *** canmallocgc=True *** for anything that
     # can malloc a GC object.
 
+    # __________ qcgc operations __________
+    'qcgc_allocate':    LLOp(canmallocgc=True),
+
     # __________ weakrefs __________
 
     'weakref_create':       LLOp(sideeffects=False, canmallocgc=True),
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -950,3 +950,11 @@
 
     def OP_QCGC_POP_ROOT(self, op):
         return 'qcgc_shadowstack_pop();'
+
+    def OP_QCGC_ALLOCATE(self, op):
+        # XXX: SET typeid
+        size = self.expr(op.args[0])
+        typeid = self.expr(op.args[1])
+        result = self.expr(op.result)
+        return ('%s = qcgc_allocate(%s);' % (result, size) +
+                '((PYPYHDR *)%s)->tid = %s;' % (result, typeid))

From pypy.commits at gmail.com  Sat Aug 20 11:41:57 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sat, 20 Aug 2016 08:41:57 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: qcgc_allocate seems to compile
Message-ID: <57b87a45.85261c0a.721b3.596d@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86350:399683dc9715
Date: 2016-08-20 17:41 +0200
http://bitbucket.org/pypy/pypy/changeset/399683dc9715/

Log:	qcgc_allocate seems to compile

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -17,7 +17,7 @@
 
     TRANSLATION_PARAMS = {}
     HDR = lltype.Struct(
-            'PYPYHDR',
+            'pypyhdr_t',
             ('hdr', rffi.COpaque('object_t', hints={"is_qcgc_header": True})),
             ('tid', lltype.Signed),
             ('hash', lltype.Signed))
diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -896,6 +896,7 @@
     eci = ExternalCompilationInfo(
             include_dirs = [library_dir],
             includes = ["qcgc.h"],
+            pre_include_bits = ["#define RPY_QCGC 1"],
             separate_module_sources = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
                 ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -957,4 +957,4 @@
         typeid = self.expr(op.args[1])
         result = self.expr(op.result)
         return ('%s = qcgc_allocate(%s);' % (result, size) +
-                '((PYPYHDR *)%s)->tid = %s;' % (result, typeid))
+                '((pypyhdr_t *)%s)->tid = %s;' % (result, typeid))
diff --git a/rpython/translator/c/src/g_prerequisite.h b/rpython/translator/c/src/g_prerequisite.h
--- a/rpython/translator/c/src/g_prerequisite.h
+++ b/rpython/translator/c/src/g_prerequisite.h
@@ -23,3 +23,11 @@
 # define RPY_LENGTH0     1       /* array decl [0] are bad */
 # define RPY_DUMMY_VARLENGTH     /* nothing */
 #endif
+
+#ifdef RPY_QCGC
+typedef struct {
+	object_t hdr;
+	int32_t tid;
+	int32_t hash;
+} pypyhdr_t;
+#endif

From pypy.commits at gmail.com  Sat Aug 20 12:23:08 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sat, 20 Aug 2016 09:23:08 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Implement malloc_varsize_clear
 (WIP)
Message-ID: <57b883ec.05371c0a.afb9f.627e@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86351:3d1e9993d643
Date: 2016-08-20 18:22 +0200
http://bitbucket.org/pypy/pypy/changeset/3d1e9993d643/

Log:	Implement malloc_varsize_clear (WIP)

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -1,6 +1,7 @@
 from rpython.memory.gc.base import GCBase
-from rpython.rtyper.lltypesystem import rffi, lltype, llgroup, llmemory
+from rpython.rtyper.lltypesystem import rffi, lltype, llgroup, llmemory, llarena
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rlib.debug import ll_assert
 
 class QCGC(GCBase):
     _alloc_flavor_ = "raw"
@@ -27,24 +28,19 @@
                                needs_finalizer=False,
                                is_finalizer_light=False,
                                contains_weakptr=False):
-        # What is the llmemory.GCREF for?
+        # XXX: What is the llmemory.GCREF for? (Assumption: return value)
+        ll_assert(not needs_finalizer, 'finalizer not supported')
+        ll_assert(not is_finalizer_light, 'light finalizer not supported')
+        ll_assert(not contains_weakptr, 'weakref not supported')
         return llop.qcgc_allocate(llmemory.GCREF, size, typeid)
-        ## XXX finalizers are ignored for now
-        ##ll_assert(not needs_finalizer, 'XXX needs_finalizer')
-        ##ll_assert(not is_finalizer_light, 'XXX is_finalizer_light')
-        #ll_assert(not contains_weakptr, 'contains_weakptr: use malloc_weakref')
-        ## XXX call optimized versions, e.g. if size < GC_NURSERY_SECTION
-        #return llop.stm_allocate(llmemory.GCREF, size, typeid16)
 
     def malloc_varsize_clear(self, typeid16, length, size, itemsize,
                              offset_to_length):
-        raise NotImplementedError
-        ## XXX be careful about overflows, and call optimized versions
-        #totalsize = size + itemsize * length
-        #totalsize = llarena.round_up_for_allocation(totalsize)
-        #obj = llop.stm_allocate(llmemory.Address, totalsize, typeid16)
-        #(obj + offset_to_length).signed[0] = length
-        #return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+        totalsize = size + itemsize * length
+        totalsize = llarena.round_up_for_allocation(totalsize)
+        obj = llop.qcgc_allocate(llmemory.Address, totalsize, typeid16)
+        (obj + offset_to_length).signed[0] = length
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     def collect(self, gen=1):
         """Do a minor (gen=0) or major (gen>0) collection."""

From pypy.commits at gmail.com  Sat Aug 20 12:36:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 09:36:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: Tests, fix
Message-ID: <57b886ff.497bc20a.7638e.476e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86352:06fb039274e4
Date: 2016-08-20 17:55 +0200
http://bitbucket.org/pypy/pypy/changeset/06fb039274e4/

Log:	Tests, fix

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -481,7 +481,7 @@
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
     def visit_kwonly(self, typ):
-        self.unwrap.append("None")
+        raise FastFuncNotSupported
 
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -809,6 +809,71 @@
         w_res = space.call_args(w_g, args)
         assert space.eq_w(w_res, space.newbytes('foo'))
 
+    def test_unwrap_spec_kwonly(self):
+        space = self.space
+        def g(space, w_x, __kwonly__, w_y):
+            return space.sub(w_x, w_y)
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w = space.wrap
+        w1 = w(1)
+
+        for i in range(4):
+            a = argument.Arguments(space, [w1, w1, w1])
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+            py.test.raises(gateway.OperationError, space.call_function, w_g,
+                           *(i * (w1,)))
+
+        args = argument.Arguments(space, [w(1)],
+                                  w_starstararg = w({'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-9))
+        args = argument.Arguments(space, [],
+                                  w_starstararg = w({'x': 2, 'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-8))
+
+    def test_unwrap_spec_kwonly_default(self):
+        space = self.space
+        @gateway.unwrap_spec(w_x2=WrappedDefault(50), y2=int)
+        def g(space, w_x1, w_x2, __kwonly__, w_y1, y2=200):
+            return space.sub(space.sub(w_x1, w_x2),
+                             space.sub(w_y1, w(y2)))
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w = space.wrap
+        w1 = w(1)
+
+        for i in range(6):
+            py.test.raises(gateway.OperationError, space.call_function, w_g,
+                           *(i * (w1,)))
+
+        def expected(x1, x2=50, y1="missing", y2=200):
+            return (x1 - x2) - (y1 - y2)
+
+        def check(*args, **kwds):
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            w_res = space.call_args(w_g, a)
+            assert space.eq_w(w_res, w(expected(*args, **kwds)))
+
+            del kwds['y1']
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+
+            args += (1234,)
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+
+        check(5,       y1=1234)
+        check(5, 1,    y1=1234)
+        check(5, x2=1, y1=1234)
+        check(5,       y1=1234, y2=343)
+        check(5, 1,    y1=1234, y2=343)
+        check(5, x2=1, y1=1234, y2=343)
+        check(x1=5,       y1=1234,       )
+        check(x1=5, x2=1, y1=1234,       )
+        check(x1=5,       y1=1234, y2=343)
+        check(x1=5, x2=1, y1=1234, y2=343)
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -379,6 +379,20 @@
         assert X().f() == 42
         """
 
+    def test_kwonlyarg_required(self):
+        """
+        def f(*, a=5, b):
+            return (a, b)
+        assert f(b=10) == (5, 10)
+        assert f(a=7, b=12) == (7, 12)
+        raises(TypeError, f)
+        raises(TypeError, f, 1)
+        raises(TypeError, f, 1, 1)
+        raises(TypeError, f, a=1)
+        raises(TypeError, f, 1, a=1)
+        raises(TypeError, f, 1, b=1)
+        """
+
     def test_extended_unpacking_short(self):
         """
         class Seq:

From pypy.commits at gmail.com  Sat Aug 20 12:36:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 09:36:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: Test and fix
Message-ID: <57b88702.85c11c0a.cb5e1.6b5a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86354:85917066f6c4
Date: 2016-08-20 18:30 +0200
http://bitbucket.org/pypy/pypy/changeset/85917066f6c4/

Log:	Test and fix

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -999,6 +999,7 @@
             elif isinstance(spec, WrappedDefault):
                 assert name.startswith('w_')
                 defaultval = spec.default_value
+                w_def = Ellipsis
 
             if defaultval is not NO_DEFAULT:
                 if name != '__args__' and name != 'args_w':
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -874,6 +874,15 @@
         check(x1=5,       y1=1234, y2=343)
         check(x1=5, x2=1, y1=1234, y2=343)
 
+    def test_unwrap_spec_kwonly_default_2(self):
+        space = self.space
+        @gateway.unwrap_spec(w_x2=WrappedDefault(50))
+        def g(space, w_x2=None):
+            return w_x2
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_res = space.call_function(w_g)
+        assert space.eq_w(w_res, space.wrap(50))
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist

From pypy.commits at gmail.com  Sat Aug 20 12:36:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 09:36:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: oops, broke translation
Message-ID: <57b88704.44ce1c0a.325f4.6718@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86355:1982fef3bbf8
Date: 2016-08-20 18:35 +0200
http://bitbucket.org/pypy/pypy/changeset/1982fef3bbf8/

Log:	oops, broke translation

diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -131,25 +131,29 @@
 
     def getdict(self, space):
         if self.lazy:
-            # Force the dictionary by calling all lazy loaders now.
-            # This also saves in self.w_initialdict a copy of all the
-            # initial values, including if they have already been
-            # modified by setdictvalue().
-            for name in self.loaders:
-                w_value = self.get(name)
-                space.setitem(self.w_dict, space.new_interned_str(name), w_value)
-            self.lazy = False
-            self.save_module_content_for_future_reload()
-            for key, w_initial_value in self.lazy_initial_values_w.items():
-                w_key = space.new_interned_str(key)
-                if w_initial_value is not None:
-                    space.setitem(self.w_initialdict, w_key, w_initial_value)
-                else:
-                    if space.finditem(self.w_initialdict, w_key) is not None:
-                        space.delitem(self.w_initialdict, w_key)
-            del self.lazy_initial_values_w
+            self._force_lazy_dict_now()
         return self.w_dict
 
+    def _force_lazy_dict_now(self):
+        # Force the dictionary by calling all lazy loaders now.
+        # This also saves in self.w_initialdict a copy of all the
+        # initial values, including if they have already been
+        # modified by setdictvalue().
+        space = self.space
+        for name in self.loaders:
+            w_value = self.get(name)
+            space.setitem(self.w_dict, space.new_interned_str(name), w_value)
+        self.lazy = False
+        self.save_module_content_for_future_reload()
+        for key, w_initial_value in self.lazy_initial_values_w.items():
+            w_key = space.new_interned_str(key)
+            if w_initial_value is not None:
+                space.setitem(self.w_initialdict, w_key, w_initial_value)
+            else:
+                if space.finditem(self.w_initialdict, w_key) is not None:
+                    space.delitem(self.w_initialdict, w_key)
+        del self.lazy_initial_values_w
+
     def _cleanup_(self):
         self.getdict(self.space)
         self.w_initialdict = None

From pypy.commits at gmail.com  Sat Aug 20 12:36:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 09:36:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: fix test,
 remove unused method
Message-ID: <57b88700.898b1c0a.1c93f.73f2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86353:827160043f5d
Date: 2016-08-20 18:17 +0200
http://bitbucket.org/pypy/pypy/changeset/827160043f5d/

Log:	fix test, remove unused method

diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -40,9 +40,6 @@
         and possibly more locals."""
         return self.signature().getallvarnames()
 
-    def getformalargcount(self):
-        return self.signature().scope_length()
-
     def getdocstring(self, space):
         return space.w_None
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -30,7 +30,7 @@
         assert sig.num_argnames() == 3
         assert sig.has_vararg()
         assert sig.has_kwarg()
-        assert sig.scope_length() == 5
+        assert sig.scope_length() == 6
         assert sig.getallvarnames() == ["a", "b", "c", "d", "kwonly", "c"]
 
     def test_eq(self):

From pypy.commits at gmail.com  Sat Aug 20 13:34:46 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 20 Aug 2016 10:34:46 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: test,
 fix for passing flags arg around
Message-ID: <57b894b6.4abf1c0a.b6879.846c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86358:8dc76d4c4aa3
Date: 2016-08-21 05:33 +1200
http://bitbucket.org/pypy/pypy/changeset/8dc76d4c4aa3/

Log:	test, fix for passing flags arg around

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -208,7 +208,7 @@
     def buffer_w(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self, flags)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
@@ -216,7 +216,8 @@
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                                space.BUF_FULL_RO)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
@@ -224,7 +225,8 @@
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                                space.BUF_FULL)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
@@ -232,7 +234,8 @@
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                                space.BUF_FULL_RO)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -346,8 +346,13 @@
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
     with lltype.scoped_alloc(Py_buffer) as pybuf:
-        # XXX flags are not in w_args?
-        flags = rffi.cast(rffi.INT_real,0)
+        _flags = 0
+        if space.len_w(w_args) > 0:
+            _flags = space.listview(w_args)[0]
+        if not isinstance(_flags, int):
+            raise oefmt(space.w_TypeError, 
+                        "non-int flags passed to getbufferproc")
+        flags = rffi.cast(rffi.INT_real,_flags)
         size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
         if widen(size) < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
@@ -356,7 +361,10 @@
         ndim = widen(pybuf.c_ndim)
         shape =   [pybuf.c_shape[i]   for i in range(ndim)]
         strides = [pybuf.c_strides[i] for i in range(ndim)]
-        format = rffi.charp2str(pybuf.c_format)
+        if pybuf.c_format:
+            format = rffi.charp2str(pybuf.c_format)
+        else:
+            format = 'B'
         return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
                             ndim=ndim, shape=shape, strides=strides,
                             itemsize=pybuf.c_itemsize,
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -106,6 +106,10 @@
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
+  if (flags == 0) {
+    PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
+    return -1;
+  }
 
   PyMyArray* self = (PyMyArray*)obj;
   view->obj = (PyObject*)self;

From pypy.commits at gmail.com  Sat Aug 20 13:45:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 10:45:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Add an assert that we get consistently the
 rpython 'str' type here, 
Message-ID: <57b89721.e2efc20a.ba596.5f4c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86359:79b3a413aab8
Date: 2016-08-20 19:44 +0200
http://bitbucket.org/pypy/pypy/changeset/79b3a413aab8/

Log:	Add an assert that we get consistently the rpython 'str' type here,
	and fix two tests that passed 'unicode' explicitly

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -68,6 +68,7 @@
     @jit.elidable
     def find_map_attr(self, name, index):
         # attr cache
+        assert type(name) is str    # utf8-encoded
         space = self.space
         cache = space.fromcache(MapAttrCache)
         SHIFT2 = r_uint.BITS - space.config.objspace.std.methodcachesizeexp
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1282,7 +1282,7 @@
             assert a == self.string2
             assert b == 2000
             if not self._str_devolves:
-                result = self.impl.getitem_str(self.string)
+                result = self.impl.getitem_str(self.string.encode('utf-8'))
             else:
                 result = self.impl.getitem(self.string)
             assert result == 1000
@@ -1293,7 +1293,7 @@
         assert self.impl.length() == 1
         assert self.impl.getitem(self.string) == 1000
         if not self._str_devolves:
-            result = self.impl.getitem_str(self.string)
+            result = self.impl.getitem_str(self.string.encode('utf-8'))
         else:
             result = self.impl.getitem(self.string)
         assert result == 1000

From pypy.commits at gmail.com  Sat Aug 20 14:04:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 11:04:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: hg merge py3k
Message-ID: <57b89bc9.82cbc20a.4def.6517@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86360:97a36a112c3c
Date: 2016-08-20 19:45 +0200
http://bitbucket.org/pypy/pypy/changeset/97a36a112c3c/

Log:	hg merge py3k

diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1282,7 +1282,7 @@
             assert a == self.string2
             assert b == 2000
             if not self._str_devolves:
-                result = self.impl.getitem_str(self.string)
+                result = self.impl.getitem_str(self.string.encode('utf-8'))
             else:
                 result = self.impl.getitem(self.string)
             assert result == 1000
@@ -1293,7 +1293,7 @@
         assert self.impl.length() == 1
         assert self.impl.getitem(self.string) == 1000
         if not self._str_devolves:
-            result = self.impl.getitem_str(self.string)
+            result = self.impl.getitem_str(self.string.encode('utf-8'))
         else:
             result = self.impl.getitem(self.string)
         assert result == 1000

From pypy.commits at gmail.com  Sat Aug 20 14:04:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 11:04:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: Put the ".decode('utf-8')"
 here, whenever we wrap an RPython-level string
Message-ID: <57b89bcb.031dc20a.34dd5.70fb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86361:0582c616e50c
Date: 2016-08-20 20:04 +0200
http://bitbucket.org/pypy/pypy/changeset/0582c616e50c/

Log:	Put the ".decode('utf-8')" here, whenever we wrap an RPython-level
	string as an app-level unicode attribute name

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -337,7 +337,7 @@
             space = self.space
             w_dict = obj.getdict(space)
             try:
-                space.delitem(w_dict, space.wrap(name))
+                space.delitem(w_dict, space.wrap(name.decode('utf-8')))
             except OperationError as ex:
                 if not ex.match(space, space.w_KeyError):
                     raise
@@ -402,7 +402,7 @@
     def materialize_r_dict(self, space, obj, dict_w):
         new_obj = self.back.materialize_r_dict(space, obj, dict_w)
         if self.index == DICT:
-            w_attr = space.wrap(self.name)
+            w_attr = space.wrap(self.name.decode('utf-8'))
             dict_w[w_attr] = obj._mapdict_read_storage(self.storageindex)
         else:
             self._copy_attr(obj, new_obj)
@@ -810,7 +810,7 @@
             raise KeyError
         key = curr.name
         w_value = self.getitem_str(w_dict, key)
-        w_key = self.space.wrap(key)
+        w_key = self.space.wrap(key.decode('utf-8'))
         self.delitem(w_dict, w_key)
         return (w_key, w_value)
 
@@ -845,7 +845,7 @@
             if curr_map:
                 self.curr_map = curr_map.back
                 attr = curr_map.name
-                w_attr = self.space.wrap(attr)
+                w_attr = self.space.wrap(attr.decode('utf-8'))
                 return w_attr
         return None
 
@@ -886,7 +886,7 @@
             if curr_map:
                 self.curr_map = curr_map.back
                 attr = curr_map.name
-                w_attr = self.space.wrap(attr)
+                w_attr = self.space.wrap(attr.decode('utf-8'))
                 return w_attr, self.w_obj.getdictvalue(self.space, attr)
         return None, None
 

From pypy.commits at gmail.com  Sat Aug 20 16:58:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 13:58:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-kwonly-builtin: close branch, ready to merge
Message-ID: <57b8c466.4fd71c0a.fa5da.bbd7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-kwonly-builtin
Changeset: r86362:20ef4376e9de
Date: 2016-08-20 22:56 +0200
http://bitbucket.org/pypy/pypy/changeset/20ef4376e9de/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Sat Aug 20 16:58:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 13:58:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge py3k-kwonly-builtin
Message-ID: <57b8c468.8f8e1c0a.2f009.ba94@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86363:aceb45156627
Date: 2016-08-20 22:57 +0200
http://bitbucket.org/pypy/pypy/changeset/aceb45156627/

Log:	hg merge py3k-kwonly-builtin

	Fix kwonly handling for built-in functions, and also probably fix
	the handling of kwonly arguments in general in the JIT

diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -40,9 +40,6 @@
         and possibly more locals."""
         return self.signature().getallvarnames()
 
-    def getformalargcount(self):
-        return self.signature().scope_length()
-
     def getdocstring(self, space):
         return space.w_None
 
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -38,7 +38,9 @@
                           'name?',
                           'w_kw_defs?']
 
-    def __init__(self, space, code, w_globals=None, defs_w=[], w_kw_defs=None,
+    w_kw_defs = None
+
+    def __init__(self, space, code, w_globals=None, defs_w=[], kw_defs_w=None,
                  closure=None, w_ann=None, forcename=None, qualname=None):
         self.space = space
         self.name = forcename or code.co_name
@@ -48,10 +50,12 @@
         self.w_func_globals = w_globals  # the globals dictionary
         self.closure = closure    # normally, list of Cell instances or None
         self.defs_w = defs_w
-        self.w_kw_defs = w_kw_defs
         self.w_func_dict = None # filled out below if needed
         self.w_module = None
         self.w_ann = w_ann
+        #
+        if kw_defs_w is not None:
+            self.init_kwdefaults_dict(kw_defs_w)
 
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
@@ -379,14 +383,29 @@
 
     def fset_func_kwdefaults(self, space, w_new):
         if space.is_w(w_new, space.w_None):
-            w_new = None
-        elif not space.isinstance_w(w_new, space.w_dict):
-            raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
-        self.w_kw_defs = w_new
+            self.w_kw_defs = None
+        else:
+            if not space.isinstance_w(w_new, space.w_dict):
+                raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
+            w_instance = self.init_kwdefaults_dict()
+            w_instance.setdict(space, w_new)
+            self.w_kw_defs = w_instance.getdict(space)
 
     def fdel_func_kwdefaults(self, space):
         self.w_kw_defs = None
 
+    def init_kwdefaults_dict(self, kw_defs_w=[]):
+        # use the mapdict logic to get at least not-too-bad JIT code
+        # from function calls with default values of kwonly arguments
+        space = self.space
+        w_class = space.fromcache(KwDefsClassCache).w_class
+        w_instance = space.call_function(w_class)
+        for w_name, w_value in kw_defs_w:
+            attr = space.unicode_w(w_name).encode('utf-8')
+            w_instance.setdictvalue(space, attr, w_value)
+        self.w_kw_defs = w_instance.getdict(space)
+        return w_instance
+
     def fget_func_doc(self, space):
         if self.w_doc is None:
             self.w_doc = self.code.getdocstring(space)
@@ -663,10 +682,12 @@
     def __init__(self, func):
         assert isinstance(func, Function)
         Function.__init__(self, func.space, func.code, func.w_func_globals,
-                          func.defs_w, None, func.closure, None, func.name)
+                          func.defs_w, None, func.closure,
+                          None, func.name)
         self.w_doc = func.w_doc
         self.w_func_dict = func.w_func_dict
         self.w_module = func.w_module
+        self.w_kw_defs = func.w_kw_defs
 
     def descr_builtinfunction__new__(space, w_subtype):
         raise oefmt(space.w_TypeError,
@@ -684,3 +705,12 @@
     else:
         code = None
     return isinstance(code, BuiltinCode)
+
+
+class KwDefsClassCache:
+    def __init__(self, space):
+        self.w_class = space.appexec([], """():
+            class KwDefs:
+                pass
+            return KwDefs
+        """)
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -28,6 +28,8 @@
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
 
+NO_DEFAULT = object()
+
 
 # internal non-translatable parts:
 class SignatureBuilder(object):
@@ -44,12 +46,21 @@
         self.argnames = argnames
         self.varargname = varargname
         self.kwargname = kwargname
+        self.kwonlyargnames = None
 
     def append(self, argname):
-        self.argnames.append(argname)
+        if self.kwonlyargnames is None:
+            self.argnames.append(argname)
+        else:
+            self.kwonlyargnames.append(argname)
+
+    def marker_kwonly(self):
+        assert self.kwonlyargnames is None
+        self.kwonlyargnames = []
 
     def signature(self):
-        return Signature(self.argnames, self.varargname, self.kwargname)
+        return Signature(self.argnames, self.varargname, self.kwargname,
+                         self.kwonlyargnames)
 
 #________________________________________________________________
 
@@ -66,13 +77,6 @@
         """NOT_RPYTHON"""
         raise NotImplementedError
 
-def kwonly(arg_unwrapper):
-    """Mark argument as keyword-only.
-
-    XXX: has no actual effect for now.
-    """
-    return arg_unwrapper
-
 
 class UnwrapSpecRecipe(object):
     "NOT_RPYTHON"
@@ -229,6 +233,11 @@
         name = int_unwrapping_space_method(typ)
         self.checked_space_method(name, app_sig)
 
+    def visit_kwonly(self, _, app_sig):
+        argname = self.orig_arg()
+        assert argname == '__kwonly__'
+        app_sig.marker_kwonly()
+
 
 class UnwrapSpec_EmitRun(UnwrapSpecEmit):
 
@@ -316,6 +325,9 @@
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
+    def visit_kwonly(self, typ):
+        self.run_args.append("None")
+
     def _make_unwrap_activation_class(self, unwrap_spec, cache={}):
         try:
             key = tuple(unwrap_spec)
@@ -468,6 +480,9 @@
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
+    def visit_kwonly(self, typ):
+        raise FastFuncNotSupported
+
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
         unwrap_info.apply_over(unwrap_spec)
@@ -563,6 +578,8 @@
                 unwrap_spec.append('args_w')
             elif argname.startswith('w_'):
                 unwrap_spec.append(W_Root)
+            elif argname == '__kwonly__':
+                unwrap_spec.append('kwonly')
             else:
                 unwrap_spec.append(None)
 
@@ -616,6 +633,8 @@
         argnames = sig.argnames
         varargname = sig.varargname
         kwargname = sig.kwargname
+        if sig.kwonlyargnames:
+            import pdb; pdb.set_trace()
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -950,64 +969,71 @@
         self.name = app_name
         self.as_classmethod = as_classmethod
 
-        if not f.func_defaults:
-            self._staticdefs = []
-        else:
-            argnames = self._code._argnames
-            defaults = f.func_defaults
-            self._staticdefs = zip(argnames[-len(defaults):], defaults)
+        argnames = self._code._argnames
+        defaults = f.func_defaults or ()
+        self._staticdefs = dict(zip(
+            argnames[len(argnames) - len(defaults):], defaults))
+
         return self
 
     def _getdefaults(self, space):
         "NOT_RPYTHON"
-        defs_w = []
-        unwrap_spec = self._code._unwrap_spec[-len(self._staticdefs):]
-        for i, (name, defaultval) in enumerate(self._staticdefs):
+        alldefs_w = {}
+        assert len(self._code._argnames) == len(self._code._unwrap_spec)
+        for name, spec in zip(self._code._argnames, self._code._unwrap_spec):
+            if name == '__kwonly__':
+                continue
+
+            defaultval = self._staticdefs.get(name, NO_DEFAULT)
+            w_def = Ellipsis
             if name.startswith('w_'):
-                assert defaultval is None, (
+                assert defaultval in (NO_DEFAULT, None), (
                     "%s: default value for '%s' can only be None, got %r; "
                     "use unwrap_spec(...=WrappedDefault(default))" % (
                     self._code.identifier, name, defaultval))
-                defs_w.append(None)
-            elif name != '__args__' and name != 'args_w':
-                spec = unwrap_spec[i]
-                if isinstance(defaultval, str) and spec not in [str]:
-                    defs_w.append(space.newbytes(defaultval))
-                else:
-                    defs_w.append(space.wrap(defaultval))
-        if self._code._unwrap_spec:
-            UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
-            if defs_w:
-                alldefs_w[-len(defs_w):] = defs_w
-            code = self._code
-            assert isinstance(code._unwrap_spec, (list, tuple))
-            assert isinstance(code._argnames, list)
-            assert len(code._unwrap_spec) == len(code._argnames)
-            for i in range(len(code._unwrap_spec)-1, -1, -1):
-                spec = code._unwrap_spec[i]
-                argname = code._argnames[i]
-                if isinstance(spec, tuple) and spec[0] is W_Root:
-                    assert False, "use WrappedDefault"
-                if isinstance(spec, WrappedDefault):
-                    default_value = spec.default_value
-                    if isinstance(default_value, str):
-                        w_default = space.newbytes(default_value)
-                    else:
-                        w_default = space.wrap(default_value)
-                    assert isinstance(w_default, W_Root)
-                    assert argname.startswith('w_')
-                    argname = argname[2:]
-                    j = self._code.sig.argnames.index(argname)
-                    assert alldefs_w[j] in (UNDEFINED, None)
-                    alldefs_w[j] = w_default
-            first_defined = 0
-            while (first_defined < len(alldefs_w) and
-                   alldefs_w[first_defined] is UNDEFINED):
-                first_defined += 1
-            defs_w = alldefs_w[first_defined:]
-            assert UNDEFINED not in defs_w
-        return defs_w
+                if defaultval is None:
+                    w_def = None
+
+            if isinstance(spec, tuple) and spec[0] is W_Root:
+                assert False, "use WrappedDefault"
+            elif isinstance(spec, WrappedDefault):
+                assert name.startswith('w_')
+                defaultval = spec.default_value
+                w_def = Ellipsis
+
+            if defaultval is not NO_DEFAULT:
+                if name != '__args__' and name != 'args_w':
+                    if w_def is Ellipsis:
+                        if isinstance(defaultval, str) and spec not in [str]:
+                            w_def = space.newbytes(defaultval)
+                        else:
+                            w_def = space.wrap(defaultval)
+                    if name.startswith('w_'):
+                        name = name[2:]
+                    alldefs_w[name] = w_def
+        #
+        # Here, 'alldefs_w' maps some argnames to their wrapped default
+        # value.  We return two lists:
+        #  - a list of defaults for positional arguments, which covers
+        #    some suffix of the sig.argnames list
+        #  - a list of pairs (w_name, w_def) for kwonly arguments
+        #
+        sig = self._code.sig
+        first_defined = 0
+        while (first_defined < len(sig.argnames) and
+               sig.argnames[first_defined] not in alldefs_w):
+            first_defined += 1
+        defs_w = [alldefs_w.pop(name) for name in sig.argnames[first_defined:]]
+
+        kw_defs_w = None
+        if alldefs_w:
+            kw_defs_w = []
+            for name, w_def in sorted(alldefs_w.items()):
+                assert name in sig.kwonlyargnames
+                w_name = space.newunicode(name.decode('utf-8'))
+                kw_defs_w.append((w_name, w_def))
+
+        return defs_w, kw_defs_w
 
     # lazy binding to space
 
@@ -1027,9 +1053,10 @@
     def build(cache, gateway):
         "NOT_RPYTHON"
         space = cache.space
-        defs = gateway._getdefaults(space) # needs to be implemented by subclass
+        defs_w, kw_defs_w = gateway._getdefaults(space)
         code = gateway._code
-        fn = FunctionWithFixedCode(space, code, None, defs, forcename=gateway.name)
+        fn = FunctionWithFixedCode(space, code, None, defs_w, kw_defs_w,
+                                   forcename=gateway.name)
         if not space.config.translating:
             fn.add_to_table()
         if gateway.as_classmethod:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1235,15 +1235,15 @@
             for i in range(len(names_w) - 1, -1, -1):
                 space.setitem(w_ann, names_w[i], self.popvalue())
         defaultarguments = self.popvalues(posdefaults)
-        w_kw_defs = None
+        kw_defs_w = None
         if kwdefaults:
-            w_kw_defs = space.newdict(strdict=True)
-            for i in range(kwdefaults - 1, -1, -1):
-                w_name = self.popvalue()
-                w_def = self.popvalue()
-                space.setitem(w_kw_defs, w_def, w_name)
+            kw_defs_w = []
+            for i in range(kwdefaults):
+                w_defvalue = self.popvalue()
+                w_defname = self.popvalue()
+                kw_defs_w.append((w_defname, w_defvalue))
         fn = function.Function(space, codeobj, self.get_w_globals(), defaultarguments,
-                               w_kw_defs, freevars, w_ann, qualname=qualname)
+                               kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
     def MAKE_FUNCTION(self, oparg, next_instr):
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -39,6 +39,7 @@
 
     def scope_length(self):
         scopelen = len(self.argnames)
+        scopelen += len(self.kwonlyargnames)
         scopelen += self.has_vararg()
         scopelen += self.has_kwarg()
         return scopelen
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -30,7 +30,7 @@
         assert sig.num_argnames() == 3
         assert sig.has_vararg()
         assert sig.has_kwarg()
-        assert sig.scope_length() == 5
+        assert sig.scope_length() == 6
         assert sig.getallvarnames() == ["a", "b", "c", "d", "kwonly", "c"]
 
     def test_eq(self):
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -47,6 +47,12 @@
         code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace, "index"])
         assert code.signature() == Signature(["index"], None, None)
 
+        def f(space, __kwonly__, w_x):
+            pass
+        code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace,
+                                                   "kwonly", W_Root])
+        assert code.signature() == Signature([], kwonlyargnames=['x'])
+
 
     def test_call(self):
         def c(space, w_x, w_y, hello_w):
@@ -753,7 +759,7 @@
         @gateway.unwrap_spec(w_x = WrappedDefault(42), y=int)
         def g(space, w_x, y):
             never_called
-        py.test.raises(AssertionError, space.wrap, gateway.interp2app_temp(g))
+        py.test.raises(KeyError, space.wrap, gateway.interp2app_temp(g))
 
     def test_unwrap_spec_default_applevel_bug2(self):
         space = self.space
@@ -803,6 +809,80 @@
         w_res = space.call_args(w_g, args)
         assert space.eq_w(w_res, space.newbytes('foo'))
 
+    def test_unwrap_spec_kwonly(self):
+        space = self.space
+        def g(space, w_x, __kwonly__, w_y):
+            return space.sub(w_x, w_y)
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w = space.wrap
+        w1 = w(1)
+
+        for i in range(4):
+            a = argument.Arguments(space, [w1, w1, w1])
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+            py.test.raises(gateway.OperationError, space.call_function, w_g,
+                           *(i * (w1,)))
+
+        args = argument.Arguments(space, [w(1)],
+                                  w_starstararg = w({'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-9))
+        args = argument.Arguments(space, [],
+                                  w_starstararg = w({'x': 2, 'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-8))
+
+    def test_unwrap_spec_kwonly_default(self):
+        space = self.space
+        @gateway.unwrap_spec(w_x2=WrappedDefault(50), y2=int)
+        def g(space, w_x1, w_x2, __kwonly__, w_y1, y2=200):
+            return space.sub(space.sub(w_x1, w_x2),
+                             space.sub(w_y1, w(y2)))
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w = space.wrap
+        w1 = w(1)
+
+        for i in range(6):
+            py.test.raises(gateway.OperationError, space.call_function, w_g,
+                           *(i * (w1,)))
+
+        def expected(x1, x2=50, y1="missing", y2=200):
+            return (x1 - x2) - (y1 - y2)
+
+        def check(*args, **kwds):
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            w_res = space.call_args(w_g, a)
+            assert space.eq_w(w_res, w(expected(*args, **kwds)))
+
+            del kwds['y1']
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+
+            args += (1234,)
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+
+        check(5,       y1=1234)
+        check(5, 1,    y1=1234)
+        check(5, x2=1, y1=1234)
+        check(5,       y1=1234, y2=343)
+        check(5, 1,    y1=1234, y2=343)
+        check(5, x2=1, y1=1234, y2=343)
+        check(x1=5,       y1=1234,       )
+        check(x1=5, x2=1, y1=1234,       )
+        check(x1=5,       y1=1234, y2=343)
+        check(x1=5, x2=1, y1=1234, y2=343)
+
+    def test_unwrap_spec_kwonly_default_2(self):
+        space = self.space
+        @gateway.unwrap_spec(w_x2=WrappedDefault(50))
+        def g(space, w_x2=None):
+            return w_x2
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_res = space.call_function(w_g)
+        assert space.eq_w(w_res, space.wrap(50))
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -379,6 +379,20 @@
         assert X().f() == 42
         """
 
+    def test_kwonlyarg_required(self):
+        """
+        def f(*, a=5, b):
+            return (a, b)
+        assert f(b=10) == (5, 10)
+        assert f(a=7, b=12) == (7, 12)
+        raises(TypeError, f)
+        raises(TypeError, f, 1)
+        raises(TypeError, f, 1, 1)
+        raises(TypeError, f, a=1)
+        raises(TypeError, f, 1, a=1)
+        raises(TypeError, f, 1, b=1)
+        """
+
     def test_extended_unpacking_short(self):
         """
         class Seq:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -15,8 +15,7 @@
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
 
-from pypy.interpreter.gateway import (
-    unwrap_spec, WrappedDefault, Unwrapper, kwonly)
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, Unwrapper
 from pypy.interpreter.error import (
     OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror)
 from pypy.interpreter.executioncontext import ExecutionContext
@@ -211,7 +210,8 @@
             "%s: %s unavailable on this platform", funcname, arg)
 
 @unwrap_spec(flags=c_int, mode=c_int, dir_fd=DirFD(rposix.HAVE_OPENAT))
-def open(space, w_path, flags, mode=0777, dir_fd=DEFAULT_DIR_FD):
+def open(space, w_path, flags, mode=0777,
+         __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """open(path, flags, mode=0o777, *, dir_fd=None)
 
 Open a file for low level IO.  Returns a file handle (integer).
@@ -428,8 +428,8 @@
 @unwrap_spec(
     path=path_or_fd(allow_fd=True),
     dir_fd=DirFD(rposix.HAVE_FSTATAT),
-    follow_symlinks=kwonly(bool))
-def stat(space, path, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
+    follow_symlinks=bool)
+def stat(space, path, __kwonly__, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
     """stat(path, *, dir_fd=None, follow_symlinks=True) -> stat result
 
 Perform a stat system call on the given path.
@@ -476,7 +476,7 @@
 @unwrap_spec(
     path=path_or_fd(allow_fd=False),
     dir_fd=DirFD(rposix.HAVE_FSTATAT))
-def lstat(space, path, dir_fd=DEFAULT_DIR_FD):
+def lstat(space, path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """lstat(path, *, dir_fd=None) -> stat result
 
 Like stat(), but do not follow symbolic links.
@@ -551,9 +551,9 @@
         raise wrap_oserror(space, e)
 
 @unwrap_spec(mode=c_int,
-    dir_fd=DirFD(rposix.HAVE_FACCESSAT), effective_ids=kwonly(bool),
-    follow_symlinks=kwonly(bool))
-def access(space, w_path, mode,
+    dir_fd=DirFD(rposix.HAVE_FACCESSAT), effective_ids=bool,
+    follow_symlinks=bool)
+def access(space, w_path, mode, __kwonly__,
         dir_fd=DEFAULT_DIR_FD, effective_ids=False, follow_symlinks=True):
     """\
 access(path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True)
@@ -626,7 +626,7 @@
         return space.wrap(rc)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
-def unlink(space, w_path, dir_fd=DEFAULT_DIR_FD):
+def unlink(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """unlink(path, *, dir_fd=None)
 
 Remove a file (same as remove()).
@@ -645,7 +645,7 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
-def remove(space, w_path, dir_fd=DEFAULT_DIR_FD):
+def remove(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """remove(path, *, dir_fd=None)
 
 Remove a file (same as unlink()).
@@ -710,7 +710,7 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_MKDIRAT))
-def mkdir(space, w_path, mode=0o777, dir_fd=DEFAULT_DIR_FD):
+def mkdir(space, w_path, mode=0o777, __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """mkdir(path, mode=0o777, *, dir_fd=None)
 
 Create a directory.
@@ -731,7 +731,7 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
-def rmdir(space, w_path, dir_fd=DEFAULT_DIR_FD):
+def rmdir(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """rmdir(path, *, dir_fd=None)
 
 Remove a directory.
@@ -898,8 +898,9 @@
     return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),
-             follow_symlinks=kwonly(bool))
-def chmod(space, w_path, mode, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
+             follow_symlinks=bool)
+def chmod(space, w_path, mode, __kwonly__,
+          dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
     """chmod(path, mode, *, dir_fd=None, follow_symlinks=True)
 
 Change the access permissions of a file.
@@ -965,7 +966,7 @@
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
-def rename(space, w_src, w_dst,
+def rename(space, w_src, w_dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD):
     """rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None)
 
@@ -989,7 +990,7 @@
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
-def replace(space, w_src, w_dst,
+def replace(space, w_src, w_dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD):
     """replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None)
 
@@ -1012,7 +1013,7 @@
         raise wrap_oserror(space, e)
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_MKFIFOAT))
-def mkfifo(space, w_path, mode=0666, dir_fd=DEFAULT_DIR_FD):
+def mkfifo(space, w_path, mode=0666, __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """mkfifo(path, mode=0o666, *, dir_fd=None)
 
 Create a FIFO (a POSIX named pipe).
@@ -1031,7 +1032,8 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(mode=c_int, device=c_int, dir_fd=DirFD(rposix.HAVE_MKNODAT))
-def mknod(space, w_filename, mode=0600, device=0, dir_fd=DEFAULT_DIR_FD):
+def mknod(space, w_filename, mode=0600, device=0,
+          __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """mknod(filename, mode=0o600, device=0, *, dir_fd=None)
 
 Create a filesystem node (file, device special file or named pipe)
@@ -1093,9 +1095,9 @@
 @unwrap_spec(
     src='fsencode', dst='fsencode',
     src_dir_fd=DirFD(rposix.HAVE_LINKAT), dst_dir_fd=DirFD(rposix.HAVE_LINKAT),
-    follow_symlinks=kwonly(bool))
+    follow_symlinks=bool)
 def link(
-        space, src, dst,
+        space, src, dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD,
         follow_symlinks=True):
     """\
@@ -1125,7 +1127,7 @@
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_SYMLINKAT))
 def symlink(space, w_src, w_dst, w_target_is_directory=None,
-        dir_fd=DEFAULT_DIR_FD):
+            __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """symlink(src, dst, target_is_directory=False, *, dir_fd=None)
 
 Create a symbolic link pointing to src named dst.
@@ -1153,7 +1155,7 @@
 @unwrap_spec(
     path=path_or_fd(allow_fd=False),
     dir_fd=DirFD(rposix.HAVE_READLINKAT))
-def readlink(space, path, dir_fd=DEFAULT_DIR_FD):
+def readlink(space, path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """readlink(path, *, dir_fd=None) -> path
 
 Return a string representing the path to which the symbolic link points.
@@ -1353,9 +1355,9 @@
 
 @unwrap_spec(
     path=path_or_fd(allow_fd=rposix.HAVE_FUTIMENS or rposix.HAVE_FUTIMES),
-    w_times=WrappedDefault(None), w_ns=kwonly(WrappedDefault(None)),
-    dir_fd=DirFD(rposix.HAVE_UTIMENSAT), follow_symlinks=kwonly(bool))
-def utime(space, path, w_times, w_ns, dir_fd=DEFAULT_DIR_FD,
+    w_times=WrappedDefault(None), w_ns=WrappedDefault(None),
+    dir_fd=DirFD(rposix.HAVE_UTIMENSAT), follow_symlinks=bool)
+def utime(space, path, w_times, __kwonly__, w_ns, dir_fd=DEFAULT_DIR_FD,
           follow_symlinks=True):
     """utime(path, times=None, *, ns=None, dir_fd=None, follow_symlinks=True)
 
@@ -1889,8 +1891,9 @@
 
 @unwrap_spec(
     uid=c_uid_t, gid=c_gid_t,
-    dir_fd=DirFD(rposix.HAVE_FCHOWNAT), follow_symlinks=kwonly(bool))
-def chown(space, w_path, uid, gid, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
+    dir_fd=DirFD(rposix.HAVE_FCHOWNAT), follow_symlinks=bool)
+def chown(space, w_path, uid, gid, __kwonly__,
+          dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
     """chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True)
 
 Change the owner and group id of path to the numeric uid and gid.
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -337,7 +337,7 @@
             space = self.space
             w_dict = obj.getdict(space)
             try:
-                space.delitem(w_dict, space.wrap(name))
+                space.delitem(w_dict, space.wrap(name.decode('utf-8')))
             except OperationError as ex:
                 if not ex.match(space, space.w_KeyError):
                     raise
@@ -402,7 +402,7 @@
     def materialize_r_dict(self, space, obj, dict_w):
         new_obj = self.back.materialize_r_dict(space, obj, dict_w)
         if self.index == DICT:
-            w_attr = space.wrap(self.name)
+            w_attr = space.wrap(self.name.decode('utf-8'))
             dict_w[w_attr] = obj._mapdict_read_storage(self.storageindex)
         else:
             self._copy_attr(obj, new_obj)
@@ -810,7 +810,7 @@
             raise KeyError
         key = curr.name
         w_value = self.getitem_str(w_dict, key)
-        w_key = self.space.wrap(key)
+        w_key = self.space.wrap(key.decode('utf-8'))
         self.delitem(w_dict, w_key)
         return (w_key, w_value)
 
@@ -845,7 +845,7 @@
             if curr_map:
                 self.curr_map = curr_map.back
                 attr = curr_map.name
-                w_attr = self.space.wrap(attr)
+                w_attr = self.space.wrap(attr.decode('utf-8'))
                 return w_attr
         return None
 
@@ -886,7 +886,7 @@
             if curr_map:
                 self.curr_map = curr_map.back
                 attr = curr_map.name
-                w_attr = self.space.wrap(attr)
+                w_attr = self.space.wrap(attr.decode('utf-8'))
                 return w_attr, self.w_obj.getdictvalue(self.space, attr)
         return None, None
 
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictObject
 from pypy.objspace.std.mapdict import *
 
@@ -873,6 +875,15 @@
         d = x.__dict__
         assert list(__pypy__.reversed_dict(d)) == list(d.keys())[::-1]
 
+    def test_nonascii_argname(self):
+        """
+        class X:
+            pass
+        x = X()
+        x.&#26085;&#26412; = 3
+        assert x.&#26085;&#26412; == 3
+        assert x.__dict__ == {'&#26085;&#26412;': 3}
+        """
 
 class AppTestWithMapDictAndCounters(object):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}

From pypy.commits at gmail.com  Sat Aug 20 17:17:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 14:17:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57b8c8de.56421c0a.ef4c7.bfba@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86364:7d8da1e4bd2c
Date: 2016-08-20 23:11 +0200
http://bitbucket.org/pypy/pypy/changeset/7d8da1e4bd2c/

Log:	hg merge py3k

diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -40,9 +40,6 @@
         and possibly more locals."""
         return self.signature().getallvarnames()
 
-    def getformalargcount(self):
-        return self.signature().scope_length()
-
     def getdocstring(self, space):
         return space.w_None
 
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -38,7 +38,9 @@
                           'name?',
                           'w_kw_defs?']
 
-    def __init__(self, space, code, w_globals=None, defs_w=[], w_kw_defs=None,
+    w_kw_defs = None
+
+    def __init__(self, space, code, w_globals=None, defs_w=[], kw_defs_w=None,
                  closure=None, w_ann=None, forcename=None, qualname=None):
         self.space = space
         self.name = forcename or code.co_name
@@ -48,10 +50,12 @@
         self.w_func_globals = w_globals  # the globals dictionary
         self.closure = closure    # normally, list of Cell instances or None
         self.defs_w = defs_w
-        self.w_kw_defs = w_kw_defs
         self.w_func_dict = None # filled out below if needed
         self.w_module = None
         self.w_ann = w_ann
+        #
+        if kw_defs_w is not None:
+            self.init_kwdefaults_dict(kw_defs_w)
 
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
@@ -379,14 +383,29 @@
 
     def fset_func_kwdefaults(self, space, w_new):
         if space.is_w(w_new, space.w_None):
-            w_new = None
-        elif not space.isinstance_w(w_new, space.w_dict):
-            raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
-        self.w_kw_defs = w_new
+            self.w_kw_defs = None
+        else:
+            if not space.isinstance_w(w_new, space.w_dict):
+                raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
+            w_instance = self.init_kwdefaults_dict()
+            w_instance.setdict(space, w_new)
+            self.w_kw_defs = w_instance.getdict(space)
 
     def fdel_func_kwdefaults(self, space):
         self.w_kw_defs = None
 
+    def init_kwdefaults_dict(self, kw_defs_w=[]):
+        # use the mapdict logic to get at least not-too-bad JIT code
+        # from function calls with default values of kwonly arguments
+        space = self.space
+        w_class = space.fromcache(KwDefsClassCache).w_class
+        w_instance = space.call_function(w_class)
+        for w_name, w_value in kw_defs_w:
+            attr = space.unicode_w(w_name).encode('utf-8')
+            w_instance.setdictvalue(space, attr, w_value)
+        self.w_kw_defs = w_instance.getdict(space)
+        return w_instance
+
     def fget_func_doc(self, space):
         if self.w_doc is None:
             self.w_doc = self.code.getdocstring(space)
@@ -663,10 +682,12 @@
     def __init__(self, func):
         assert isinstance(func, Function)
         Function.__init__(self, func.space, func.code, func.w_func_globals,
-                          func.defs_w, None, func.closure, None, func.name)
+                          func.defs_w, None, func.closure,
+                          None, func.name)
         self.w_doc = func.w_doc
         self.w_func_dict = func.w_func_dict
         self.w_module = func.w_module
+        self.w_kw_defs = func.w_kw_defs
 
     def descr_builtinfunction__new__(space, w_subtype):
         raise oefmt(space.w_TypeError,
@@ -684,3 +705,12 @@
     else:
         code = None
     return isinstance(code, BuiltinCode)
+
+
+class KwDefsClassCache:
+    def __init__(self, space):
+        self.w_class = space.appexec([], """():
+            class KwDefs:
+                pass
+            return KwDefs
+        """)
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -28,6 +28,8 @@
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
 
+NO_DEFAULT = object()
+
 
 # internal non-translatable parts:
 class SignatureBuilder(object):
@@ -44,12 +46,21 @@
         self.argnames = argnames
         self.varargname = varargname
         self.kwargname = kwargname
+        self.kwonlyargnames = None
 
     def append(self, argname):
-        self.argnames.append(argname)
+        if self.kwonlyargnames is None:
+            self.argnames.append(argname)
+        else:
+            self.kwonlyargnames.append(argname)
+
+    def marker_kwonly(self):
+        assert self.kwonlyargnames is None
+        self.kwonlyargnames = []
 
     def signature(self):
-        return Signature(self.argnames, self.varargname, self.kwargname)
+        return Signature(self.argnames, self.varargname, self.kwargname,
+                         self.kwonlyargnames)
 
 #________________________________________________________________
 
@@ -66,13 +77,6 @@
         """NOT_RPYTHON"""
         raise NotImplementedError
 
-def kwonly(arg_unwrapper):
-    """Mark argument as keyword-only.
-
-    XXX: has no actual effect for now.
-    """
-    return arg_unwrapper
-
 
 class UnwrapSpecRecipe(object):
     "NOT_RPYTHON"
@@ -229,6 +233,11 @@
         name = int_unwrapping_space_method(typ)
         self.checked_space_method(name, app_sig)
 
+    def visit_kwonly(self, _, app_sig):
+        argname = self.orig_arg()
+        assert argname == '__kwonly__'
+        app_sig.marker_kwonly()
+
 
 class UnwrapSpec_EmitRun(UnwrapSpecEmit):
 
@@ -316,6 +325,9 @@
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
+    def visit_kwonly(self, typ):
+        self.run_args.append("None")
+
     def _make_unwrap_activation_class(self, unwrap_spec, cache={}):
         try:
             key = tuple(unwrap_spec)
@@ -468,6 +480,9 @@
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
+    def visit_kwonly(self, typ):
+        raise FastFuncNotSupported
+
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
         unwrap_info.apply_over(unwrap_spec)
@@ -563,6 +578,8 @@
                 unwrap_spec.append('args_w')
             elif argname.startswith('w_'):
                 unwrap_spec.append(W_Root)
+            elif argname == '__kwonly__':
+                unwrap_spec.append('kwonly')
             else:
                 unwrap_spec.append(None)
 
@@ -616,6 +633,8 @@
         argnames = sig.argnames
         varargname = sig.varargname
         kwargname = sig.kwargname
+        if sig.kwonlyargnames:
+            import pdb; pdb.set_trace()
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -950,64 +969,71 @@
         self.name = app_name
         self.as_classmethod = as_classmethod
 
-        if not f.func_defaults:
-            self._staticdefs = []
-        else:
-            argnames = self._code._argnames
-            defaults = f.func_defaults
-            self._staticdefs = zip(argnames[-len(defaults):], defaults)
+        argnames = self._code._argnames
+        defaults = f.func_defaults or ()
+        self._staticdefs = dict(zip(
+            argnames[len(argnames) - len(defaults):], defaults))
+
         return self
 
     def _getdefaults(self, space):
         "NOT_RPYTHON"
-        defs_w = []
-        unwrap_spec = self._code._unwrap_spec[-len(self._staticdefs):]
-        for i, (name, defaultval) in enumerate(self._staticdefs):
+        alldefs_w = {}
+        assert len(self._code._argnames) == len(self._code._unwrap_spec)
+        for name, spec in zip(self._code._argnames, self._code._unwrap_spec):
+            if name == '__kwonly__':
+                continue
+
+            defaultval = self._staticdefs.get(name, NO_DEFAULT)
+            w_def = Ellipsis
             if name.startswith('w_'):
-                assert defaultval is None, (
+                assert defaultval in (NO_DEFAULT, None), (
                     "%s: default value for '%s' can only be None, got %r; "
                     "use unwrap_spec(...=WrappedDefault(default))" % (
                     self._code.identifier, name, defaultval))
-                defs_w.append(None)
-            elif name != '__args__' and name != 'args_w':
-                spec = unwrap_spec[i]
-                if isinstance(defaultval, str) and spec not in [str]:
-                    defs_w.append(space.newbytes(defaultval))
-                else:
-                    defs_w.append(space.wrap(defaultval))
-        if self._code._unwrap_spec:
-            UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
-            if defs_w:
-                alldefs_w[-len(defs_w):] = defs_w
-            code = self._code
-            assert isinstance(code._unwrap_spec, (list, tuple))
-            assert isinstance(code._argnames, list)
-            assert len(code._unwrap_spec) == len(code._argnames)
-            for i in range(len(code._unwrap_spec)-1, -1, -1):
-                spec = code._unwrap_spec[i]
-                argname = code._argnames[i]
-                if isinstance(spec, tuple) and spec[0] is W_Root:
-                    assert False, "use WrappedDefault"
-                if isinstance(spec, WrappedDefault):
-                    default_value = spec.default_value
-                    if isinstance(default_value, str):
-                        w_default = space.newbytes(default_value)
-                    else:
-                        w_default = space.wrap(default_value)
-                    assert isinstance(w_default, W_Root)
-                    assert argname.startswith('w_')
-                    argname = argname[2:]
-                    j = self._code.sig.argnames.index(argname)
-                    assert alldefs_w[j] in (UNDEFINED, None)
-                    alldefs_w[j] = w_default
-            first_defined = 0
-            while (first_defined < len(alldefs_w) and
-                   alldefs_w[first_defined] is UNDEFINED):
-                first_defined += 1
-            defs_w = alldefs_w[first_defined:]
-            assert UNDEFINED not in defs_w
-        return defs_w
+                if defaultval is None:
+                    w_def = None
+
+            if isinstance(spec, tuple) and spec[0] is W_Root:
+                assert False, "use WrappedDefault"
+            elif isinstance(spec, WrappedDefault):
+                assert name.startswith('w_')
+                defaultval = spec.default_value
+                w_def = Ellipsis
+
+            if defaultval is not NO_DEFAULT:
+                if name != '__args__' and name != 'args_w':
+                    if w_def is Ellipsis:
+                        if isinstance(defaultval, str) and spec not in [str]:
+                            w_def = space.newbytes(defaultval)
+                        else:
+                            w_def = space.wrap(defaultval)
+                    if name.startswith('w_'):
+                        name = name[2:]
+                    alldefs_w[name] = w_def
+        #
+        # Here, 'alldefs_w' maps some argnames to their wrapped default
+        # value.  We return two lists:
+        #  - a list of defaults for positional arguments, which covers
+        #    some suffix of the sig.argnames list
+        #  - a list of pairs (w_name, w_def) for kwonly arguments
+        #
+        sig = self._code.sig
+        first_defined = 0
+        while (first_defined < len(sig.argnames) and
+               sig.argnames[first_defined] not in alldefs_w):
+            first_defined += 1
+        defs_w = [alldefs_w.pop(name) for name in sig.argnames[first_defined:]]
+
+        kw_defs_w = None
+        if alldefs_w:
+            kw_defs_w = []
+            for name, w_def in sorted(alldefs_w.items()):
+                assert name in sig.kwonlyargnames
+                w_name = space.newunicode(name.decode('utf-8'))
+                kw_defs_w.append((w_name, w_def))
+
+        return defs_w, kw_defs_w
 
     # lazy binding to space
 
@@ -1027,9 +1053,10 @@
     def build(cache, gateway):
         "NOT_RPYTHON"
         space = cache.space
-        defs = gateway._getdefaults(space) # needs to be implemented by subclass
+        defs_w, kw_defs_w = gateway._getdefaults(space)
         code = gateway._code
-        fn = FunctionWithFixedCode(space, code, None, defs, forcename=gateway.name)
+        fn = FunctionWithFixedCode(space, code, None, defs_w, kw_defs_w,
+                                   forcename=gateway.name)
         if not space.config.translating:
             fn.add_to_table()
         if gateway.as_classmethod:
@@ -1167,15 +1194,15 @@
             source = source[source.find('\n') + 1:].lstrip()
         assert source.startswith("def "), "can only transform functions"
         source = source[4:]
-        import __future__
-        if flags & __future__.CO_FUTURE_DIVISION:
-            prefix += "from __future__ import division\n"
-        if flags & __future__.CO_FUTURE_ABSOLUTE_IMPORT:
-            prefix += "from __future__ import absolute_import\n"
-        if flags & __future__.CO_FUTURE_PRINT_FUNCTION:
-            prefix += "from __future__ import print_function\n"
-        if flags & __future__.CO_FUTURE_UNICODE_LITERALS:
-            prefix += "from __future__ import unicode_literals\n"
+        # The following flags have no effect any more in app-level code
+        # (i.e. they are always on anyway), and have been removed:
+        #    CO_FUTURE_DIVISION
+        #    CO_FUTURE_ABSOLUTE_IMPORT
+        #    CO_FUTURE_PRINT_FUNCTION
+        #    CO_FUTURE_UNICODE_LITERALS
+        # Original code was, for each of these flags:
+        #    if flags & __future__.CO_xxx:
+        #        prefix += "from __future__ import yyy\n"
     p = source.find('(')
     assert p >= 0
     funcname = source[:p].strip()
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -19,6 +19,7 @@
         """ NOT_RPYTHON """
         Module.__init__(self, space, w_name)
         self.lazy = True
+        self.lazy_initial_values_w = {}
         self.__class__.buildloaders()
         self.loaders = self.loaders.copy()    # copy from the class to the inst
         self.submodules_w = []
@@ -57,22 +58,11 @@
             if not self.lazy and self.w_initialdict is None:
                 self.save_module_content_for_future_reload()
 
-    def save_module_content_for_future_reload(self, save_all=False):
-        # Because setdictvalue is unable to immediately load all attributes
-        # (due to an importlib bootstrapping problem), this method needs to be
-        # able to support saving the content of a module's dict without
-        # requiring that the entire dict already be loaded. To support that
-        # properly, when updating the dict, we must be careful to never
-        # overwrite the value of a key already in w_initialdict. (So as to avoid
-        # overriding the builtin value with a user-provided value)
-        if self.space.is_none(self.w_initialdict) or save_all:
-            self.w_initialdict = self.space.call_method(self.w_dict, 'copy')
-        else:
-            w_items = self.space.call_method(self.w_dict, 'items')
-            for w_item in self.space.iteriterable(w_items):
-                w_key, w_value = self.space.fixedview(w_item, expected_length=2)
-                if not self.space.contains_w(self.w_initialdict, w_key):
-                    self.space.setitem(self.w_initialdict, w_key, w_value)
+    def save_module_content_for_future_reload(self):
+        # Save the current dictionary in w_initialdict, for future
+        # reloads.  This forces the dictionary if needed.
+        w_dict = self.getdict(self.space)
+        self.w_initialdict = self.space.call_method(w_dict, 'copy')
 
     @classmethod
     def get_applevel_name(cls):
@@ -101,9 +91,13 @@
         return w_value
 
     def setdictvalue(self, space, attr, w_value):
-        if self.lazy:
-            self._load_lazily(space, attr)
-            self.save_module_content_for_future_reload()
+        if self.lazy and attr not in self.lazy_initial_values_w:
+            # in lazy mode, the first time an attribute changes,
+            # we save away the old (initial) value.  This allows
+            # a future getdict() call to build the correct
+            # self.w_initialdict, containing the initial value.
+            w_initial_value = self._load_lazily(space, attr)
+            self.lazy_initial_values_w[attr] = w_initial_value
         space.setitem_str(self.w_dict, attr, w_value)
         return True
 
@@ -137,13 +131,29 @@
 
     def getdict(self, space):
         if self.lazy:
-            for name in self.loaders:
-                w_value = self.get(name)
-                space.setitem(self.w_dict, space.new_interned_str(name), w_value)
-            self.lazy = False
-            self.save_module_content_for_future_reload()
+            self._force_lazy_dict_now()
         return self.w_dict
 
+    def _force_lazy_dict_now(self):
+        # Force the dictionary by calling all lazy loaders now.
+        # This also saves in self.w_initialdict a copy of all the
+        # initial values, including if they have already been
+        # modified by setdictvalue().
+        space = self.space
+        for name in self.loaders:
+            w_value = self.get(name)
+            space.setitem(self.w_dict, space.new_interned_str(name), w_value)
+        self.lazy = False
+        self.save_module_content_for_future_reload()
+        for key, w_initial_value in self.lazy_initial_values_w.items():
+            w_key = space.new_interned_str(key)
+            if w_initial_value is not None:
+                space.setitem(self.w_initialdict, w_key, w_initial_value)
+            else:
+                if space.finditem(self.w_initialdict, w_key) is not None:
+                    space.delitem(self.w_initialdict, w_key)
+        del self.lazy_initial_values_w
+
     def _cleanup_(self):
         self.getdict(self.space)
         self.w_initialdict = None
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1288,15 +1288,15 @@
             for i in range(len(names_w) - 1, -1, -1):
                 space.setitem(w_ann, names_w[i], self.popvalue())
         defaultarguments = self.popvalues(posdefaults)
-        w_kw_defs = None
+        kw_defs_w = None
         if kwdefaults:
-            w_kw_defs = space.newdict(strdict=True)
-            for i in range(kwdefaults - 1, -1, -1):
-                w_name = self.popvalue()
-                w_def = self.popvalue()
-                space.setitem(w_kw_defs, w_def, w_name)
+            kw_defs_w = []
+            for i in range(kwdefaults):
+                w_defvalue = self.popvalue()
+                w_defname = self.popvalue()
+                kw_defs_w.append((w_defname, w_defvalue))
         fn = function.Function(space, codeobj, self.get_w_globals(), defaultarguments,
-                               w_kw_defs, freevars, w_ann, qualname=qualname)
+                               kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
     def MAKE_FUNCTION(self, oparg, next_instr):
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -39,6 +39,7 @@
 
     def scope_length(self):
         scopelen = len(self.argnames)
+        scopelen += len(self.kwonlyargnames)
         scopelen += self.has_vararg()
         scopelen += self.has_kwarg()
         return scopelen
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -30,7 +30,7 @@
         assert sig.num_argnames() == 3
         assert sig.has_vararg()
         assert sig.has_kwarg()
-        assert sig.scope_length() == 5
+        assert sig.scope_length() == 6
         assert sig.getallvarnames() == ["a", "b", "c", "d", "kwonly", "c"]
 
     def test_eq(self):
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -47,6 +47,12 @@
         code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace, "index"])
         assert code.signature() == Signature(["index"], None, None)
 
+        def f(space, __kwonly__, w_x):
+            pass
+        code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace,
+                                                   "kwonly", W_Root])
+        assert code.signature() == Signature([], kwonlyargnames=['x'])
+
 
     def test_call(self):
         def c(space, w_x, w_y, hello_w):
@@ -753,7 +759,7 @@
         @gateway.unwrap_spec(w_x = WrappedDefault(42), y=int)
         def g(space, w_x, y):
             never_called
-        py.test.raises(AssertionError, space.wrap, gateway.interp2app_temp(g))
+        py.test.raises(KeyError, space.wrap, gateway.interp2app_temp(g))
 
     def test_unwrap_spec_default_applevel_bug2(self):
         space = self.space
@@ -803,6 +809,80 @@
         w_res = space.call_args(w_g, args)
         assert space.eq_w(w_res, space.newbytes('foo'))
 
+    def test_unwrap_spec_kwonly(self):
+        space = self.space
+        def g(space, w_x, __kwonly__, w_y):
+            return space.sub(w_x, w_y)
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w = space.wrap
+        w1 = w(1)
+
+        for i in range(4):
+            a = argument.Arguments(space, [w1, w1, w1])
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+            py.test.raises(gateway.OperationError, space.call_function, w_g,
+                           *(i * (w1,)))
+
+        args = argument.Arguments(space, [w(1)],
+                                  w_starstararg = w({'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-9))
+        args = argument.Arguments(space, [],
+                                  w_starstararg = w({'x': 2, 'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-8))
+
+    def test_unwrap_spec_kwonly_default(self):
+        space = self.space
+        @gateway.unwrap_spec(w_x2=WrappedDefault(50), y2=int)
+        def g(space, w_x1, w_x2, __kwonly__, w_y1, y2=200):
+            return space.sub(space.sub(w_x1, w_x2),
+                             space.sub(w_y1, w(y2)))
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w = space.wrap
+        w1 = w(1)
+
+        for i in range(6):
+            py.test.raises(gateway.OperationError, space.call_function, w_g,
+                           *(i * (w1,)))
+
+        def expected(x1, x2=50, y1="missing", y2=200):
+            return (x1 - x2) - (y1 - y2)
+
+        def check(*args, **kwds):
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            w_res = space.call_args(w_g, a)
+            assert space.eq_w(w_res, w(expected(*args, **kwds)))
+
+            del kwds['y1']
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+
+            args += (1234,)
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+
+        check(5,       y1=1234)
+        check(5, 1,    y1=1234)
+        check(5, x2=1, y1=1234)
+        check(5,       y1=1234, y2=343)
+        check(5, 1,    y1=1234, y2=343)
+        check(5, x2=1, y1=1234, y2=343)
+        check(x1=5,       y1=1234,       )
+        check(x1=5, x2=1, y1=1234,       )
+        check(x1=5,       y1=1234, y2=343)
+        check(x1=5, x2=1, y1=1234, y2=343)
+
+    def test_unwrap_spec_kwonly_default_2(self):
+        space = self.space
+        @gateway.unwrap_spec(w_x2=WrappedDefault(50))
+        def g(space, w_x2=None):
+            return w_x2
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_res = space.call_function(w_g)
+        assert space.eq_w(w_res, space.wrap(50))
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -435,6 +435,20 @@
         assert X().f() == 42
         """
 
+    def test_kwonlyarg_required(self):
+        """
+        def f(*, a=5, b):
+            return (a, b)
+        assert f(b=10) == (5, 10)
+        assert f(a=7, b=12) == (7, 12)
+        raises(TypeError, f)
+        raises(TypeError, f, 1)
+        raises(TypeError, f, 1, 1)
+        raises(TypeError, f, a=1)
+        raises(TypeError, f, 1, a=1)
+        raises(TypeError, f, 1, b=1)
+        """
+
     def test_extended_unpacking_short(self):
         """
         class Seq:
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -131,7 +131,7 @@
 
 @unwrap_spec(w_module=MixedModule)
 def save_module_content_for_future_reload(space, w_module):
-    w_module.save_module_content_for_future_reload(save_all=True)
+    w_module.save_module_content_for_future_reload()
 
 def set_code_callback(space, w_callable):
     cache = space.fromcache(CodeHookCache)
diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -16,11 +16,10 @@
     @staticmethod
     def _compile_bootstrap_module(space, name, w_name, w_dict):
         """NOT_RPYTHON"""
-        ec = space.getexecutioncontext()
         with open(os.path.join(lib_python, 'importlib', name + '.py')) as fp:
             source = fp.read()
         pathname = "<frozen importlib.%s>" % name
-        code_w = ec.compiler.compile(source, pathname, 'exec', 0)
+        code_w = Module._cached_compile(space, source, pathname, 'exec', 0)
         space.setitem(w_dict, space.wrap('__name__'), w_name)
         space.setitem(w_dict, space.wrap('__builtins__'),
                       space.wrap(space.builtin))
@@ -43,6 +42,31 @@
 
         self.w_import = space.wrap(interp_import.import_with_frames_removed)
 
+    @staticmethod
+    def _cached_compile(space, source, *args):
+        from rpython.config.translationoption import CACHE_DIR
+        from pypy.module.marshal import interp_marshal
+
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_bootstrap')
+        try:
+            if space.config.translating:
+                raise IOError("don't use the cache when translating pypy")
+            with open(cachename, 'rb') as f:
+                previous = f.read(len(source) + 1)
+                if previous != source + '\x00':
+                    raise IOError("source changed")
+                w_bin = space.newbytes(f.read())
+                code_w = interp_marshal.loads(space, w_bin)
+        except IOError:
+            # must (re)compile the source
+            ec = space.getexecutioncontext()
+            code_w = ec.compiler.compile(source, *args)
+            w_bin = interp_marshal.dumps(space, code_w, space.wrap(2))
+            content = source + '\x00' + space.bytes_w(w_bin)
+            with open(cachename, 'wb') as f:
+                f.write(content)
+        return code_w
+
     def startup(self, space):
         """Copy our __import__ to builtins."""
         w_install = self.getdictvalue(space, '_install')
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -327,7 +327,8 @@
             i += 1
             if ch == '"':
                 content_utf8 = builder.build()
-                content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
+                content_unicode = unicodehelper.decode_utf8(
+                    self.space, content_utf8, allow_surrogates=True)
                 self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.wrap(content_unicode)
@@ -374,7 +375,8 @@
                    # this point
         #
         uchr = runicode.code_to_unichr(val)     # may be a surrogate pair again
-        utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
+        utf8_ch = unicodehelper.encode_utf8(
+            self.space, uchr, allow_surrogates=True)
         builder.append(utf8_ch)
         return i
 
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -10,10 +10,10 @@
     assert dec.skip_whitespace(8) == len(s)
     dec.close()
 
-    
+
 
 class AppTest(object):
-    spaceconfig = {"objspace.usemodules._pypyjson": True}
+    spaceconfig = {"usemodules": ['_pypyjson']}
 
     def test_raise_on_bytes(self):
         import _pypyjson
@@ -40,7 +40,7 @@
         raises(ValueError, _pypyjson.loads, 'fa')
         raises(ValueError, _pypyjson.loads, 'f')
         raises(ValueError, _pypyjson.loads, 'falXX')
-        
+
 
     def test_decode_string(self):
         import _pypyjson
@@ -69,7 +69,7 @@
         import _pypyjson
         assert _pypyjson.loads(r'"\\"') == '\\'
         assert _pypyjson.loads(r'"\""') == '"'
-        assert _pypyjson.loads(r'"\/"') == '/'       
+        assert _pypyjson.loads(r'"\/"') == '/'
         assert _pypyjson.loads(r'"\b"') == '\b'
         assert _pypyjson.loads(r'"\f"') == '\f'
         assert _pypyjson.loads(r'"\n"') == '\n'
@@ -85,7 +85,7 @@
         import _pypyjson
         s = r'"hello\nworld' # missing the trailing "
         raises(ValueError, "_pypyjson.loads(s)")
-        
+
     def test_escape_sequence_unicode(self):
         import _pypyjson
         s = r'"\u1234"'
@@ -166,7 +166,7 @@
     def test_decode_object_nonstring_key(self):
         import _pypyjson
         raises(ValueError, "_pypyjson.loads('{42: 43}')")
-        
+
     def test_decode_array(self):
         import _pypyjson
         assert _pypyjson.loads('[]') == []
@@ -183,7 +183,7 @@
         res = _pypyjson.loads('"z\\ud834\\udd20x"')
         assert res == expected
 
-    def test_surrogate_pair(self):
+    def test_lone_surrogate(self):
         import _pypyjson
         json = '{"a":"\\uD83D"}'
         res = _pypyjson.loads(json)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -15,8 +15,7 @@
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
 
-from pypy.interpreter.gateway import (
-    unwrap_spec, WrappedDefault, Unwrapper, kwonly)
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, Unwrapper
 from pypy.interpreter.error import (
     OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror)
 from pypy.interpreter.executioncontext import ExecutionContext
@@ -211,7 +210,8 @@
             "%s: %s unavailable on this platform", funcname, arg)
 
 @unwrap_spec(flags=c_int, mode=c_int, dir_fd=DirFD(rposix.HAVE_OPENAT))
-def open(space, w_path, flags, mode=0777, dir_fd=DEFAULT_DIR_FD):
+def open(space, w_path, flags, mode=0777,
+         __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """open(path, flags, mode=0o777, *, dir_fd=None)
 
 Open a file for low level IO.  Returns a file handle (integer).
@@ -428,8 +428,8 @@
 @unwrap_spec(
     path=path_or_fd(allow_fd=True),
     dir_fd=DirFD(rposix.HAVE_FSTATAT),
-    follow_symlinks=kwonly(bool))
-def stat(space, path, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
+    follow_symlinks=bool)
+def stat(space, path, __kwonly__, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
     """stat(path, *, dir_fd=None, follow_symlinks=True) -> stat result
 
 Perform a stat system call on the given path.
@@ -476,7 +476,7 @@
 @unwrap_spec(
     path=path_or_fd(allow_fd=False),
     dir_fd=DirFD(rposix.HAVE_FSTATAT))
-def lstat(space, path, dir_fd=DEFAULT_DIR_FD):
+def lstat(space, path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """lstat(path, *, dir_fd=None) -> stat result
 
 Like stat(), but do not follow symbolic links.
@@ -551,9 +551,9 @@
         raise wrap_oserror(space, e)
 
 @unwrap_spec(mode=c_int,
-    dir_fd=DirFD(rposix.HAVE_FACCESSAT), effective_ids=kwonly(bool),
-    follow_symlinks=kwonly(bool))
-def access(space, w_path, mode,
+    dir_fd=DirFD(rposix.HAVE_FACCESSAT), effective_ids=bool,
+    follow_symlinks=bool)
+def access(space, w_path, mode, __kwonly__,
         dir_fd=DEFAULT_DIR_FD, effective_ids=False, follow_symlinks=True):
     """\
 access(path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True)
@@ -626,7 +626,7 @@
         return space.wrap(rc)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
-def unlink(space, w_path, dir_fd=DEFAULT_DIR_FD):
+def unlink(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """unlink(path, *, dir_fd=None)
 
 Remove a file (same as remove()).
@@ -645,7 +645,7 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
-def remove(space, w_path, dir_fd=DEFAULT_DIR_FD):
+def remove(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """remove(path, *, dir_fd=None)
 
 Remove a file (same as unlink()).
@@ -710,7 +710,7 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_MKDIRAT))
-def mkdir(space, w_path, mode=0o777, dir_fd=DEFAULT_DIR_FD):
+def mkdir(space, w_path, mode=0o777, __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """mkdir(path, mode=0o777, *, dir_fd=None)
 
 Create a directory.
@@ -731,7 +731,7 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
-def rmdir(space, w_path, dir_fd=DEFAULT_DIR_FD):
+def rmdir(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """rmdir(path, *, dir_fd=None)
 
 Remove a directory.
@@ -901,8 +901,9 @@
     return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),
-             follow_symlinks=kwonly(bool))
-def chmod(space, w_path, mode, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
+             follow_symlinks=bool)
+def chmod(space, w_path, mode, __kwonly__,
+          dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
     """chmod(path, mode, *, dir_fd=None, follow_symlinks=True)
 
 Change the access permissions of a file.
@@ -968,7 +969,7 @@
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
-def rename(space, w_src, w_dst,
+def rename(space, w_src, w_dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD):
     """rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None)
 
@@ -992,7 +993,7 @@
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
-def replace(space, w_src, w_dst,
+def replace(space, w_src, w_dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD):
     """replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None)
 
@@ -1015,7 +1016,7 @@
         raise wrap_oserror(space, e)
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_MKFIFOAT))
-def mkfifo(space, w_path, mode=0666, dir_fd=DEFAULT_DIR_FD):
+def mkfifo(space, w_path, mode=0666, __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """mkfifo(path, mode=0o666, *, dir_fd=None)
 
 Create a FIFO (a POSIX named pipe).
@@ -1034,7 +1035,8 @@
         raise wrap_oserror2(space, e, w_path)
 
 @unwrap_spec(mode=c_int, device=c_int, dir_fd=DirFD(rposix.HAVE_MKNODAT))
-def mknod(space, w_filename, mode=0600, device=0, dir_fd=DEFAULT_DIR_FD):
+def mknod(space, w_filename, mode=0600, device=0,
+          __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """mknod(filename, mode=0o600, device=0, *, dir_fd=None)
 
 Create a filesystem node (file, device special file or named pipe)
@@ -1096,9 +1098,9 @@
 @unwrap_spec(
     src='fsencode', dst='fsencode',
     src_dir_fd=DirFD(rposix.HAVE_LINKAT), dst_dir_fd=DirFD(rposix.HAVE_LINKAT),
-    follow_symlinks=kwonly(bool))
+    follow_symlinks=bool)
 def link(
-        space, src, dst,
+        space, src, dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD,
         follow_symlinks=True):
     """\
@@ -1128,7 +1130,7 @@
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_SYMLINKAT))
 def symlink(space, w_src, w_dst, w_target_is_directory=None,
-        dir_fd=DEFAULT_DIR_FD):
+            __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
     """symlink(src, dst, target_is_directory=False, *, dir_fd=None)
 
 Create a symbolic link pointing to src named dst.
@@ -1156,7 +1158,7 @@
 @unwrap_spec(
     path=path_or_fd(allow_fd=False),
     dir_fd=DirFD(rposix.HAVE_READLINKAT))
-def readlink(space, path, dir_fd=DEFAULT_DIR_FD):
+def readlink(space, path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
     """readlink(path, *, dir_fd=None) -> path
 
 Return a string representing the path to which the symbolic link points.
@@ -1356,9 +1358,9 @@
 
 @unwrap_spec(
     path=path_or_fd(allow_fd=rposix.HAVE_FUTIMENS or rposix.HAVE_FUTIMES),
-    w_times=WrappedDefault(None), w_ns=kwonly(WrappedDefault(None)),
-    dir_fd=DirFD(rposix.HAVE_UTIMENSAT), follow_symlinks=kwonly(bool))
-def utime(space, path, w_times, w_ns, dir_fd=DEFAULT_DIR_FD,
+    w_times=WrappedDefault(None), w_ns=WrappedDefault(None),
+    dir_fd=DirFD(rposix.HAVE_UTIMENSAT), follow_symlinks=bool)
+def utime(space, path, w_times, __kwonly__, w_ns, dir_fd=DEFAULT_DIR_FD,
           follow_symlinks=True):
     """utime(path, times=None, *, ns=None, dir_fd=None, follow_symlinks=True)
 
@@ -1892,8 +1894,9 @@
 
 @unwrap_spec(
     uid=c_uid_t, gid=c_gid_t,
-    dir_fd=DirFD(rposix.HAVE_FCHOWNAT), follow_symlinks=kwonly(bool))
-def chown(space, w_path, uid, gid, dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
+    dir_fd=DirFD(rposix.HAVE_FCHOWNAT), follow_symlinks=bool)
+def chown(space, w_path, uid, gid, __kwonly__,
+          dir_fd=DEFAULT_DIR_FD, follow_symlinks=True):
     """chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True)
 
 Change the owner and group id of path to the numeric uid and gid.
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -14,6 +14,13 @@
         """NOT_RPYTHON""" # because parent __init__ isn't
         if space.config.translating:
             del self.__class__.interpleveldefs['pypy_getudir']
+            del self.__class__.appleveldefs['stdin']
+            del self.__class__.appleveldefs['__stdin__']
+            del self.__class__.appleveldefs['stdout']
+            del self.__class__.appleveldefs['__stdout__']
+            del self.__class__.appleveldefs['stderr']
+            del self.__class__.appleveldefs['__stderr__']
+
         super(Module, self).__init__(space, w_name)
         self.recursionlimit = 100
         self.defaultencoding = "utf-8"
@@ -99,6 +106,15 @@
         'flags'                 : 'app.null_sysflags',
         '_xoptions'             : 'app.null__xoptions',
         'implementation'        : 'app.implementation',
+
+        # these six attributes are here only during tests;
+        # they are removed before translation
+        'stdin'                 : 'std_test.stdin',
+        '__stdin__'             : 'std_test.stdin',
+        'stdout'                : 'std_test.stdout',
+        '__stdout__'            : 'std_test.stdout',
+        'stderr'                : 'std_test.stderr',
+        '__stderr__'            : 'std_test.stderr',
     }
 
     def startup(self, space):
@@ -123,28 +139,12 @@
         space = self.space
 
         if not space.config.translating:
-            from pypy.module.sys.interp_encoding import _getfilesystemencoding
-            self.filesystemencoding = _getfilesystemencoding(space)
-
-        if not space.config.translating:
-            # Install standard streams for tests that don't call app_main.
-            # Always use line buffering, even for tests that capture
-            # standard descriptors.
-            space.appexec([], """():
-                import sys, io
-                sys.stdin = sys.__stdin__ = io.open(0, "r", encoding="ascii",
-                                                    closefd=False)
-                sys.stdin.buffer.raw.name = "<stdin>"
-                sys.stdout = sys.__stdout__ = io.open(1, "w", encoding="ascii",
-                                                      buffering=1,
-                                                      closefd=False)
-                sys.stdout.buffer.raw.name = "<stdout>"
-                sys.stderr = sys.__stderr__ = io.open(2, "w", encoding="ascii",
-                                                      errors="backslashreplace",
-                                                      buffering=1,
-                                                      closefd=False)
-                sys.stderr.buffer.raw.name = "<stderr>"
-               """)
+            ##from pypy.module.sys.interp_encoding import _getfilesystemencoding
+            ##self.filesystemencoding = _getfilesystemencoding(space)
+            # XXX the two lines above take a few seconds to run whenever
+            # we initialize the space; for tests, use a simpler version
+            from pypy.module.sys.interp_encoding import base_encoding
+            self.filesystemencoding = space.wrap(base_encoding)
 
     def flush_std_files(self, space):
         w_stdout = space.sys.getdictvalue(space, 'stdout')
diff --git a/pypy/module/sys/std_test.py b/pypy/module/sys/std_test.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/std_test.py
@@ -0,0 +1,19 @@
+# Install standard streams for tests that don't call app_main.  Always
+# use line buffering, even for tests that capture standard descriptors.
+
+import io
+
+stdin = io.open(0, "r", encoding="ascii",
+                closefd=False)
+stdin.buffer.raw.name = "<stdin>"
+
+stdout = io.open(1, "w", encoding="ascii",
+                 buffering=1,
+                 closefd=False)
+stdout.buffer.raw.name = "<stdout>"
+
+stderr = io.open(2, "w", encoding="ascii",
+                 errors="backslashreplace",
+                 buffering=1,
+                 closefd=False)
+stderr.buffer.raw.name = "<stderr>"
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -68,6 +68,7 @@
     @jit.elidable
     def find_map_attr(self, name, index):
         # attr cache
+        assert type(name) is str    # utf8-encoded
         space = self.space
         cache = space.fromcache(MapAttrCache)
         SHIFT2 = r_uint.BITS - space.config.objspace.std.methodcachesizeexp
@@ -336,7 +337,7 @@
             space = self.space
             w_dict = obj.getdict(space)
             try:
-                space.delitem(w_dict, space.wrap(name))
+                space.delitem(w_dict, space.wrap(name.decode('utf-8')))
             except OperationError as ex:
                 if not ex.match(space, space.w_KeyError):
                     raise
@@ -401,7 +402,7 @@
     def materialize_r_dict(self, space, obj, dict_w):
         new_obj = self.back.materialize_r_dict(space, obj, dict_w)
         if self.index == DICT:
-            w_attr = space.wrap(self.name)
+            w_attr = space.wrap(self.name.decode('utf-8'))
             dict_w[w_attr] = obj._mapdict_read_storage(self.storageindex)
         else:
             self._copy_attr(obj, new_obj)
@@ -809,7 +810,7 @@
             raise KeyError
         key = curr.name
         w_value = self.getitem_str(w_dict, key)
-        w_key = self.space.wrap(key)
+        w_key = self.space.wrap(key.decode('utf-8'))
         self.delitem(w_dict, w_key)
         return (w_key, w_value)
 
@@ -844,7 +845,7 @@
             if curr_map:
                 self.curr_map = curr_map.back
                 attr = curr_map.name
-                w_attr = self.space.wrap(attr)
+                w_attr = self.space.wrap(attr.decode('utf-8'))
                 return w_attr
         return None
 
@@ -885,7 +886,7 @@
             if curr_map:
                 self.curr_map = curr_map.back
                 attr = curr_map.name
-                w_attr = self.space.wrap(attr)
+                w_attr = self.space.wrap(attr.decode('utf-8'))
                 return w_attr, self.w_obj.getdictvalue(self.space, attr)
         return None, None
 
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1282,7 +1282,7 @@
             assert a == self.string2
             assert b == 2000
             if not self._str_devolves:
-                result = self.impl.getitem_str(self.string)
+                result = self.impl.getitem_str(self.string.encode('utf-8'))
             else:
                 result = self.impl.getitem(self.string)
             assert result == 1000
@@ -1293,7 +1293,7 @@
         assert self.impl.length() == 1
         assert self.impl.getitem(self.string) == 1000
         if not self._str_devolves:
-            result = self.impl.getitem_str(self.string)
+            result = self.impl.getitem_str(self.string.encode('utf-8'))
         else:
             result = self.impl.getitem(self.string)
         assert result == 1000
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictObject
 from pypy.objspace.std.mapdict import *
 
@@ -873,6 +875,15 @@
         d = x.__dict__
         assert list(__pypy__.reversed_dict(d)) == list(d.keys())[::-1]
 
+    def test_nonascii_argname(self):
+        """
+        class X:
+            pass
+        x = X()
+        x.&#26085;&#26412; = 3
+        assert x.&#26085;&#26412; == 3
+        assert x.__dict__ == {'&#26085;&#26412;': 3}
+        """
 
 class AppTestWithMapDictAndCounters(object):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}

From pypy.commits at gmail.com  Sat Aug 20 17:18:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 Aug 2016 14:18:49 -0700 (PDT)
Subject: [pypy-commit] pypy rpython-deque: Close, abandon
Message-ID: <57b8c939.81a2c20a.9de76.a980@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-deque
Changeset: r86365:f07260687448
Date: 2016-08-20 23:18 +0200
http://bitbucket.org/pypy/pypy/changeset/f07260687448/

Log:	Close, abandon


From pypy.commits at gmail.com  Sat Aug 20 22:20:42 2016
From: pypy.commits at gmail.com (wlav)
Date: Sat, 20 Aug 2016 19:20:42 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: checkpoint: first stab at
 resurrecting the fast path with better cffi integration
Message-ID: <57b90ffa.c398c20a.80f8e.f93f@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86366:1ff61f3c0347
Date: 2016-08-20 19:16 -0700
http://bitbucket.org/pypy/pypy/changeset/1ff61f3c0347/

Log:	checkpoint: first stab at resurrecting the fast path with better
	cffi integration

diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -7,8 +7,7 @@
 #import cint_capi as backend
 
 from pypy.module.cppyy.capi.capi_types import C_SCOPE, C_TYPE, C_OBJECT,\
-   C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX,\
-   C_METHPTRGETTER, C_METHPTRGETTER_PTR
+   C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX, C_FUNC_PTR
 
 identify  = backend.identify
 pythonize = backend.pythonize
@@ -186,15 +185,15 @@
 def c_call_o(space, method, cppobj, nargs, args, cppclass):
     return _c_call_o(method, cppobj, nargs, args, cppclass.handle)
 
-_c_get_methptr_getter = rffi.llexternal(
-    "cppyy_get_methptr_getter",
-    [C_SCOPE, C_INDEX], C_METHPTRGETTER_PTR,
+_c_get_function_address = rffi.llexternal(
+    "cppyy_get_function_address",
+    [C_SCOPE, C_INDEX], C_FUNC_PTR,
     releasegil=ts_reflect,
     compilation_info=backend.eci,
     elidable_function=True,
     random_effects_on_gcobjs=False)
-def c_get_methptr_getter(space, cppscope, index):
-    return _c_get_methptr_getter(cppscope.handle, index)
+def c_get_function_address(space, cppscope, index):
+    return _c_get_function_address(cppscope.handle, index)
 
 # handling of function argument buffer ---------------------------------------
 _c_allocate_function_args = rffi.llexternal(
diff --git a/pypy/module/cppyy/capi/capi_types.py b/pypy/module/cppyy/capi/capi_types.py
--- a/pypy/module/cppyy/capi/capi_types.py
+++ b/pypy/module/cppyy/capi/capi_types.py
@@ -18,5 +18,4 @@
 C_INDEX_ARRAY = rffi.LONGP
 WLAVC_INDEX = rffi.LONG
 
-C_METHPTRGETTER = lltype.FuncType([C_OBJECT], rffi.VOIDP)
-C_METHPTRGETTER_PTR = lltype.Ptr(C_METHPTRGETTER)
+C_FUNC_PTR = rffi.VOIDP
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -10,7 +10,7 @@
 from pypy.module._cffi_backend import ctypefunc, ctypeprim, cdataobj, misc
 
 from pypy.module.cppyy.capi.capi_types import C_SCOPE, C_TYPE, C_OBJECT,\
-   C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX, C_METHPTRGETTER_PTR
+   C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX, C_FUNC_PTR
 
 
 reflection_library = 'libcppyy_backend.so'
@@ -154,7 +154,7 @@
             'constructor'  : ([c_method, c_object, c_int, c_voidp],   c_object),
             'call_o'       : ([c_method, c_object, c_int, c_voidp, c_type],     c_object),
 
-            'get_methptr_getter'       : ([c_scope, c_index],         c_voidp), # TODO: verify
+            'get_function_address'     : ([c_scope, c_index],         c_voidp), # TODO: verify
 
             # handling of function argument buffer
             'allocate_function_args'   : ([c_int],                    c_voidp),
@@ -362,10 +362,10 @@
     args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs), _Arg(h=cppclass.handle)]
     return _cdata_to_cobject(space, call_capi(space, 'call_o', args))
 
-def c_get_methptr_getter(space, cppscope, index):
+def c_get_function_address(space, cppscope, index):
     args = [_Arg(h=cppscope.handle), _Arg(l=index)]
-    return rffi.cast(C_METHPTRGETTER_PTR,
-        _cdata_to_ptr(space, call_capi(space, 'get_methptr_getter', args)))
+    return rffi.cast(C_FUNC_PTR,
+        _cdata_to_ptr(space, call_capi(space, 'get_function_address', args)))
 
 # handling of function argument buffer ---------------------------------------
 def c_allocate_function_args(space, size):
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -81,6 +81,7 @@
     libffitype  = jit_libffi.types.sint
     c_type      = rffi.INT
     c_ptrtype   = rffi.INTP
+    ctype_name  = 'int'
 
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(self.c_type, space.c_int_w(w_obj))
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -12,8 +12,8 @@
     typedef cppyy_scope_t cppyy_type_t;
     typedef unsigned long cppyy_object_t;
     typedef unsigned long cppyy_method_t;
-    typedef long cppyy_index_t;
-    typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t);
+    typedef long          cppyy_index_t;
+    typedef void*         cppyy_funcaddr_t;
 
     /* name to opaque C++ scope representation -------------------------------- */
     RPY_EXTERN
@@ -67,7 +67,7 @@
     cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type);
 
     RPY_EXTERN
-    cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx);
+    cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t scope, cppyy_index_t idx);
 
     /* handling of function argument buffer ----------------------------------- */
     RPY_EXTERN
diff --git a/pypy/module/cppyy/include/cpp_cppyy.h b/pypy/module/cppyy/include/cpp_cppyy.h
--- a/pypy/module/cppyy/include/cpp_cppyy.h
+++ b/pypy/module/cppyy/include/cpp_cppyy.h
@@ -29,7 +29,7 @@
    typedef ptrdiff_t   TCppMethod_t;
 
    typedef Long_t      TCppIndex_t;
-   typedef void* (*TCppMethPtrGetter_t)( TCppObject_t );
+   typedef void*       TCppFuncAddr_t;
 
 // name to opaque C++ scope representation -----------------------------------
    TCppIndex_t GetNumScopes( TCppScope_t parent );
@@ -67,7 +67,7 @@
    void         CallDestructor( TCppType_t type, TCppObject_t self );
    TCppObject_t CallO( TCppMethod_t method, TCppObject_t self, void* args, TCppType_t result_type );
 
-   TCppMethPtrGetter_t GetMethPtrGetter( TCppScope_t scope, TCppIndex_t imeth );
+   TCppFuncAddr_t GetFunctionAddress( TCppScope_t scope, TCppIndex_t imeth );
 
 // handling of function argument buffer --------------------------------------
    void*  AllocateFunctionArgs( size_t nargs );
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -7,9 +7,11 @@
 
 from rpython.rtyper.lltypesystem import rffi, lltype, llmemory
 
-from rpython.rlib import jit, rdynload, rweakref
+from rpython.rlib import jit, rdynload, rweakref, rgc
 from rpython.rlib import jit_libffi, clibffi
+from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 
+from pypy.module._cffi_backend import ctypefunc, newtype
 from pypy.module.cppyy import converter, executor, helper
 
 
@@ -177,7 +179,7 @@
         self.converters = None
         self.executor = None
         self.cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
-        self._funcaddr = lltype.nullptr(rffi.VOIDP.TO)
+        self._funcaddr = lltype.nullptr(capi.C_FUNC_PTR.TO)
         self.uses_local = False
 
     @staticmethod
@@ -263,8 +265,51 @@
                 self.space, cif_descr, self._funcaddr, buffer)
         finally:
             lltype.free(buffer, flavor='raw')
+            keepalive_until_here(args_w)
         return w_res
 
+    # from ctypefunc; have my own version for annotater purposes and to disable
+    # memory tracking (method live time is longer than the tests)
+    @jit.dont_look_inside
+    def _rawallocate(self, builder):
+        builder.space = self.space
+
+        # compute the total size needed in the CIF_DESCRIPTION buffer
+        builder.nb_bytes = 0
+        builder.bufferp = lltype.nullptr(rffi.CCHARP.TO)
+        builder.fb_build()
+
+        # allocate the buffer
+        if we_are_translated():
+            rawmem = lltype.malloc(rffi.CCHARP.TO, builder.nb_bytes,
+                                   flavor='raw', track_allocation=False)
+            rawmem = rffi.cast(jit_libffi.CIF_DESCRIPTION_P, rawmem)
+        else:
+            # gross overestimation of the length below, but too bad
+            rawmem = lltype.malloc(jit_libffi.CIF_DESCRIPTION_P.TO, builder.nb_bytes,
+                                   flavor='raw', track_allocation=False)
+
+        # the buffer is automatically managed from the W_CTypeFunc instance
+        self.cif_descr = rawmem
+
+        # call again fb_build() to really build the libffi data structures
+        builder.bufferp = rffi.cast(rffi.CCHARP, rawmem)
+        builder.fb_build()
+        assert builder.bufferp == rffi.ptradd(rffi.cast(rffi.CCHARP, rawmem),
+                                              builder.nb_bytes)
+
+        # fill in the 'exchange_*' fields
+        builder.fb_build_exchange(rawmem)
+
+        # fill in the extra fields
+        builder.fb_extra_fields(rawmem)
+
+        # call libffi's ffi_prep_cif() function
+        res = jit_libffi.jit_ffi_prep_cif(rawmem)
+        if res != clibffi.FFI_OK:
+            raise oefmt(self.space.w_SystemError,
+                        "libffi failed to build this function type")
+
     def _setup(self, cppthis):
         self.converters = [converter.get_converter(self.space, arg_type, arg_dflt)
                                for arg_type, arg_dflt in self.arg_defs]
@@ -279,78 +324,30 @@
         # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
-        methgetter = capi.c_get_methptr_getter(self.space, self.scope, self.index)
-        if methgetter and cppthis:      # methods only for now
-            cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
+        self._funcaddr = capi.c_get_function_address(self.space, self.scope, self.index)
+        if self._funcaddr and cppthis:      # methods only for now
+            # argument type specification (incl. cppthis)
+            fargs = []
+            fargs.append(newtype.new_pointer_type(self.space, newtype.new_void_type(self.space)))
+            for i, conv in enumerate(self.converters):
+                 if not conv.libffitype:
+                     raise FastCallNotPossible
+                 fargs.append(newtype.new_primitive_type(self.space, conv.ctype_name))
+            fresult = newtype.new_primitive_type(self.space, self.executor.ctype_name)
+
+            # the following is derived from _cffi_backend.ctypefunc
+            builder = ctypefunc.CifDescrBuilder(fargs[:], fresult, clibffi.FFI_DEFAULT_ABI)
             try:
-                funcaddr = methgetter(rffi.cast(capi.C_OBJECT, cppthis))
-                self._funcaddr = rffi.cast(rffi.VOIDP, funcaddr)
-
-                nargs = len(self.arg_defs) + 1                   # +1: cppthis
-
-                # memory block for CIF description (note: not tracked as the life
-                # time of methods is normally the duration of the application)
-                size = llmemory.sizeof(jit_libffi.CIF_DESCRIPTION, nargs)
-
-                # allocate the buffer
-                cif_descr = lltype.malloc(jit_libffi.CIF_DESCRIPTION_P.TO,
-                                          llmemory.raw_malloc_usage(size),
-                                          flavor='raw', track_allocation=False)
-
-                # array of 'ffi_type*' values, one per argument
-                size = rffi.sizeof(jit_libffi.FFI_TYPE_P) * nargs
-                atypes = lltype.malloc(rffi.CCHARP.TO, llmemory.raw_malloc_usage(size),
-                                       flavor='raw', track_allocation=False)
-                cif_descr.atypes = rffi.cast(jit_libffi.FFI_TYPE_PP, atypes)
-
-                # argument type specification
-                cif_descr.atypes[0] = jit_libffi.types.pointer   # cppthis
-                for i, conv in enumerate(self.converters):
-                    if not conv.libffitype:
-                        raise FastCallNotPossible
-                    cif_descr.atypes[i+1] = conv.libffitype
-
-                # result type specification
-                cif_descr.rtype = self.executor.libffitype
-
-                # exchange ---
-
-                # first, enough room for an array of 'nargs' pointers
-                exchange_offset = rffi.sizeof(rffi.CCHARP) * nargs
-                exchange_offset = (exchange_offset + 7) & ~7     # alignment
-                cif_descr.exchange_result = exchange_offset
-
-                # then enough room for the result, rounded up to sizeof(ffi_arg)
-                exchange_offset += max(rffi.getintfield(cif_descr.rtype, 'c_size'),
-                                       jit_libffi.SIZE_OF_FFI_ARG)
-
-                # loop over args
-                for i in range(nargs):
-                    exchange_offset = (exchange_offset + 7) & ~7 # alignment
-                    cif_descr.exchange_args[i] = exchange_offset
-                    exchange_offset += rffi.getintfield(cif_descr.atypes[i], 'c_size')
-
-                # store the exchange data size
-                cif_descr.exchange_size = exchange_offset
-
-                # --- exchange
-
-                # extra
-                cif_descr.abi = clibffi.FFI_DEFAULT_ABI
-                cif_descr.nargs = len(self.arg_defs) + 1         # +1: cppthis
-
-                res = jit_libffi.jit_ffi_prep_cif(cif_descr)
-                if res != clibffi.FFI_OK:
-                    raise FastCallNotPossible
-
-            except Exception as e:
-                if cif_descr:
-                    lltype.free(cif_descr.atypes, flavor='raw', track_allocation=False)
-                    lltype.free(cif_descr, flavor='raw', track_allocation=False)
-                cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
-                self._funcaddr = lltype.nullptr(rffi.VOIDP.TO)
-
-            self.cif_descr = cif_descr
+                self._rawallocate(builder)
+            except OperationError as e:
+                if not e.match(self.space, self.space.w_NotImplementedError):
+                    raise
+                # else, eat the NotImplementedError.  We will get the
+                # exception if we see an actual call
+                if self.cif_descr:   # should not be True, but you never know
+                    lltype.free(self.cif_descr, flavor='raw')
+                    self.cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
+                raise FastCallNotPossible
 
     @jit.unroll_safe
     def prepare_arguments(self, args_w, call_local):
@@ -394,9 +391,9 @@
             total_arg_priority += p
         return total_arg_priority
 
+    @rgc.must_be_light_finalizer
     def __del__(self):
         if self.cif_descr:
-            lltype.free(self.cif_descr.atypes, flavor='raw')
             lltype.free(self.cif_descr, flavor='raw')
 
     def __repr__(self):
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -25,9 +25,11 @@
 // Standard
 #include <assert.h>
 #include <algorithm>     // for std::count
+#include <dlfcn.h>
 #include <map>
 #include <set>
 #include <sstream>
+#include <stdlib.h>      // for getenv
 
 // temp
 #include <iostream>
@@ -59,6 +61,9 @@
 static std::set< std::string > gSmartPtrTypes =
    { "auto_ptr", "shared_ptr", "weak_ptr", "unique_ptr" };
 
+// configuration
+static bool gEnableFastPath = true;
+
 
 // global initialization -----------------------------------------------------
 namespace {
@@ -76,6 +81,8 @@
       g_classrefs.push_back(TClassRef("std"));
       // add a dummy global to refer to as null at index 0
       g_globalvars.push_back( nullptr );
+      // disable fast path if requested
+      if (getenv("CPPYY_DISABLE_FASTPATH")) gEnableFastPath = false;
    }
 
    ~ApplicationStarter() {
@@ -519,10 +526,11 @@
    return (TCppObject_t)0;
 }
 
-Cppyy::TCppMethPtrGetter_t Cppyy::GetMethPtrGetter(
-      TCppScope_t /* scope */, TCppIndex_t /* imeth */ )
+Cppyy::TCppFuncAddr_t Cppyy::GetFunctionAddress( TCppScope_t scope, TCppIndex_t imeth )
 {
-   return (TCppMethPtrGetter_t)0;
+   if (!gEnableFastPath) return (TCppFuncAddr_t)nullptr;
+   TFunction* f = type_get_method( scope, imeth );
+   return (TCppFuncAddr_t)dlsym(RTLD_DEFAULT, f->GetMangledName());
 }
 
 
@@ -1223,8 +1231,8 @@
     return cppyy_object_t(Cppyy::CallO(method, (void*)self, &parvec, result_type));
 }
 
-cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx) {
-    return cppyy_methptrgetter_t(Cppyy::GetMethPtrGetter(scope, idx));
+cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t scope, cppyy_index_t idx) {
+    return cppyy_funcaddr_t(Cppyy::GetFunctionAddress(scope, idx));
 }
 
 
diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py
--- a/pypy/module/cppyy/test/test_pythonify.py
+++ b/pypy/module/cppyy/test/test_pythonify.py
@@ -77,10 +77,11 @@
         """Test object and method calls."""
         import cppyy
         example01_class = cppyy.gbl.example01
-        assert example01_class.getCount() == 0
+        #assert example01_class.getCount() == 0
         instance = example01_class(7)
-        assert example01_class.getCount() == 1
+        #assert example01_class.getCount() == 1
         res = instance.addDataToInt(4)
+        return
         assert res == 11
         res = instance.addDataToInt(-4)
         assert res == 3
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -3,14 +3,17 @@
 from rpython.rlib.objectmodel import specialize, instantiate
 from rpython.rlib import rarithmetic, jit
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper import llinterp
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
 
-from pypy.module.cppyy import interp_cppyy, capi
+from pypy.module.cppyy import interp_cppyy, capi, executor
 # These tests are for the backend that support the fast path only.
 if capi.identify() == 'CINT':
     py.test.skip("CINT does not support fast path")
 elif capi.identify() == 'loadable_capi':
     py.test.skip("can not currently use FakeSpace with _cffi_backend")
+elif os.getenv("CPPYY_DISABLE_FASTPATH"):
+    py.test.skip("fast path is disabled by CPPYY_DISABLE_FASTPATH envar")
 
 # load cpyext early, or its global vars are counted as leaks in the test
 # (note that the module is not otherwise used in the test itself)
@@ -29,6 +32,23 @@
     return rffi.ptradd(ptr, offset)
 capi.exchange_address = _opaque_exchange_address
 
+# add missing alt_errno (??)
+def get_tlobj(self):
+    try:
+        return self._tlobj
+    except AttributeError:
+        from rpython.rtyper.lltypesystem import rffi
+        PERRNO = rffi.CArrayPtr(rffi.INT)
+        fake_p_errno = lltype.malloc(PERRNO.TO, 1, flavor='raw', zero=True,
+                                     track_allocation=False)
+        self._tlobj = {'RPY_TLOFS_p_errno': fake_p_errno,
+                       'RPY_TLOFS_alt_errno': rffi.cast(rffi.INT, 0),
+                       #'thread_ident': ...,
+                       }
+        return self._tlobj
+llinterp.LLInterpreter.get_tlobj = get_tlobj
+
+
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict.so"))
 
@@ -83,15 +103,21 @@
     def perform(self, executioncontext, frame):
         pass
 
+class FakeState(object):
+    def __init__(self, space):
+        self.slowcalls = 0
+
 class FakeSpace(object):
     fake = True
 
-    w_ValueError = FakeException("ValueError")
-    w_TypeError = FakeException("TypeError")
-    w_AttributeError = FakeException("AttributeError")
-    w_ReferenceError = FakeException("ReferenceError")
+    w_AttributeError      = FakeException("AttributeError")
+    w_KeyError            = FakeException("KeyError")
     w_NotImplementedError = FakeException("NotImplementedError")
-    w_RuntimeError = FakeException("RuntimeError")
+    w_ReferenceError      = FakeException("ReferenceError")
+    w_RuntimeError        = FakeException("RuntimeError")
+    w_SystemError         = FakeException("SystemError")
+    w_TypeError           = FakeException("TypeError")
+    w_ValueError          = FakeException("ValueError")
 
     w_None = None
     w_str = FakeType("str")
@@ -105,6 +131,12 @@
         self.config = dummy()
         self.config.translating = False
 
+        # kill calls to c_call_i (i.e. slow path)
+        def c_call_i(space, cppmethod, cppobject, nargs, args):
+            assert not "slow path called"
+            return capi.c_call_i(space, cppmethod, cppobject, nargs, args)
+        executor.get_executor(self, 'int').__class__.c_stubcall = staticmethod(c_call_i)
+
     def issequence_w(self, w_obj):
         return True
 
@@ -210,8 +242,8 @@
         f()
         space = FakeSpace()
         result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
-        # TODO: this currently succeeds even as there is no fast path implemented?!
-        self.check_jitcell_token_count(1)
+        self.check_jitcell_token_count(1)   # same for fast and slow path??
+        # rely on replacement of capi calls to raise exception instead (see FakeSpace.__init__)
 
     def test01_simple(self):
         """Test fast path being taken for methods"""

From pypy.commits at gmail.com  Sun Aug 21 01:34:35 2016
From: pypy.commits at gmail.com (wlav)
Date: Sat, 20 Aug 2016 22:34:35 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: capitulate on __dir__ for now
Message-ID: <57b93d6b.8aacc20a.9e40d.1a52@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86367:b0a2d7ad8562
Date: 2016-08-20 22:30 -0700
http://bitbucket.org/pypy/pypy/changeset/b0a2d7ad8562/

Log:	capitulate on __dir__ for now

diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -832,13 +832,7 @@
         return datamember
 
     def _find_datamembers(self):
-        num_datamembers = capi.c_num_datamembers(self.space, self)
-        for i in range(num_datamembers):
-            if not capi.c_is_publicdata(self.space, self, i):
-                continue
-            datamember_name = capi.c_datamember_name(self.space, self, i)
-            if not datamember_name in self.datamembers:
-                self._make_datamember(datamember_name, i)
+        pass       # force lazy lookups in namespaces
 
     def find_overload(self, meth_name):
         indices = capi.c_method_indices_from_name(self.space, self, meth_name)
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -132,7 +132,19 @@
 Cppyy::TCppIndex_t Cppyy::GetNumScopes( TCppScope_t scope )
 {
    TClassRef& cr = type_from_handle( scope );
-   if ( cr.GetClass() ) return 0;   // not supported if not at global scope
+   if ( cr.GetClass() ) {
+   // this is expensive, but this function is only ever called for __dir__
+   // TODO: rewrite __dir__ on the C++ side for a single loop
+       std::string s = GetFinalName( scope ); s += "::";
+       gClassTable->Init(); 
+       const int N = gClassTable->Classes();
+       int total = 0;
+       for ( int i = 0; i < N; ++i ) {
+           if ( strncmp( gClassTable->Next(), s.c_str(), s.size() ) == 0 )
+              total += 1;
+       }
+       return total;
+   }
    assert( scope == (TCppScope_t)GLOBAL_HANDLE );
    return gClassTable->Classes();
 }
@@ -141,7 +153,22 @@
 {
 // Retrieve the scope name of the scope indexed with iscope in parent.
    TClassRef& cr = type_from_handle( parent );
-   if ( cr.GetClass() ) return 0;   // not supported if not at global scope
+   if ( cr.GetClass() ) {
+   // this is expensive (quadratic in number of classes), but only ever called for __dir__
+   // TODO: rewrite __dir__ on the C++ side for a single loop
+       std::string s = GetFinalName( parent ); s += "::";
+       gClassTable->Init();
+       const int N = gClassTable->Classes();
+       int match = 0;
+       for ( int i = 0; i < N; ++i ) {
+           char* cname = gClassTable->Next();
+           if ( strncmp( cname, s.c_str(), s.size() ) == 0 && match++ == iscope ) {
+              std::string ret( cname+ s.size() );
+              return ret.substr(0, ret.find( "::" ) ); // TODO: may mean duplicates
+           }
+       }
+       // should never get here ... fall through will fail on assert below
+   }
    assert( parent == (TCppScope_t)GLOBAL_HANDLE );
    std::string name = gClassTable->At( iscope );
    if ( name.find("::") == std::string::npos )
diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py
--- a/pypy/module/cppyy/test/test_fragile.py
+++ b/pypy/module/cppyy/test/test_fragile.py
@@ -228,8 +228,10 @@
 
             assert 'nested1' in members          # namespace
 
-        assert 'fglobal' in members          # function
-        assert 'gI'in members                # variable
+        # TODO: think this through ... probably want this, but interferes with
+        # the (new) policy of lazy lookups
+        #assert 'fglobal' in members          # function
+        #assert 'gI'in members                # variable
 
     def test12_imports(self):
         """Test ability to import from namespace (or fail with ImportError)"""

From pypy.commits at gmail.com  Sun Aug 21 04:28:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 01:28:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Eh,
 I don't know how I missed that __kwdefaults__ could simply be a
Message-ID: <57b96618.44ce1c0a.325f4.57d9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86368:bee473720d24
Date: 2016-08-21 10:24 +0200
http://bitbucket.org/pypy/pypy/changeset/bee473720d24/

Log:	Eh, I don't know how I missed that __kwdefaults__ could simply be a
	moduledict

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -387,24 +387,18 @@
         else:
             if not space.isinstance_w(w_new, space.w_dict):
                 raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
-            w_instance = self.init_kwdefaults_dict()
-            w_instance.setdict(space, w_new)
-            self.w_kw_defs = w_instance.getdict(space)
+            self.w_kw_defs = w_new
 
     def fdel_func_kwdefaults(self, space):
         self.w_kw_defs = None
 
-    def init_kwdefaults_dict(self, kw_defs_w=[]):
-        # use the mapdict logic to get at least not-too-bad JIT code
-        # from function calls with default values of kwonly arguments
+    def init_kwdefaults_dict(self, kw_defs_w):
+        # use the moduledict logic to get normally-constant entries
         space = self.space
-        w_class = space.fromcache(KwDefsClassCache).w_class
-        w_instance = space.call_function(w_class)
+        w_dict = space.newdict(module=True)
         for w_name, w_value in kw_defs_w:
-            attr = space.unicode_w(w_name).encode('utf-8')
-            w_instance.setdictvalue(space, attr, w_value)
-        self.w_kw_defs = w_instance.getdict(space)
-        return w_instance
+            space.setitem(w_dict, w_name, w_value)
+        self.w_kw_defs = w_dict
 
     def fget_func_doc(self, space):
         if self.w_doc is None:
@@ -705,12 +699,3 @@
     else:
         code = None
     return isinstance(code, BuiltinCode)
-
-
-class KwDefsClassCache:
-    def __init__(self, space):
-        self.w_class = space.appexec([], """():
-            class KwDefs:
-                pass
-            return KwDefs
-        """)

From pypy.commits at gmail.com  Sun Aug 21 04:28:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 01:28:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: translation fix
Message-ID: <57b9661a.e129c20a.f2ff2.3c9c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86369:0a4a546854ad
Date: 2016-08-21 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/0a4a546854ad/

Log:	translation fix

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -68,7 +68,7 @@
     @jit.elidable
     def find_map_attr(self, name, index):
         # attr cache
-        assert type(name) is str    # utf8-encoded
+        assert isinstance(name, str)    # utf8-encoded
         space = self.space
         cache = space.fromcache(MapAttrCache)
         SHIFT2 = r_uint.BITS - space.config.objspace.std.methodcachesizeexp

From pypy.commits at gmail.com  Sun Aug 21 04:28:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 01:28:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57b9663d.041f1c0a.6f5fb.5e64@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86370:72416bdbe656
Date: 2016-08-21 10:28 +0200
http://bitbucket.org/pypy/pypy/changeset/72416bdbe656/

Log:	hg merge py3k

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -387,24 +387,18 @@
         else:
             if not space.isinstance_w(w_new, space.w_dict):
                 raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
-            w_instance = self.init_kwdefaults_dict()
-            w_instance.setdict(space, w_new)
-            self.w_kw_defs = w_instance.getdict(space)
+            self.w_kw_defs = w_new
 
     def fdel_func_kwdefaults(self, space):
         self.w_kw_defs = None
 
-    def init_kwdefaults_dict(self, kw_defs_w=[]):
-        # use the mapdict logic to get at least not-too-bad JIT code
-        # from function calls with default values of kwonly arguments
+    def init_kwdefaults_dict(self, kw_defs_w):
+        # use the moduledict logic to get normally-constant entries
         space = self.space
-        w_class = space.fromcache(KwDefsClassCache).w_class
-        w_instance = space.call_function(w_class)
+        w_dict = space.newdict(module=True)
         for w_name, w_value in kw_defs_w:
-            attr = space.unicode_w(w_name).encode('utf-8')
-            w_instance.setdictvalue(space, attr, w_value)
-        self.w_kw_defs = w_instance.getdict(space)
-        return w_instance
+            space.setitem(w_dict, w_name, w_value)
+        self.w_kw_defs = w_dict
 
     def fget_func_doc(self, space):
         if self.w_doc is None:
@@ -705,12 +699,3 @@
     else:
         code = None
     return isinstance(code, BuiltinCode)
-
-
-class KwDefsClassCache:
-    def __init__(self, space):
-        self.w_class = space.appexec([], """():
-            class KwDefs:
-                pass
-            return KwDefs
-        """)
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -68,7 +68,7 @@
     @jit.elidable
     def find_map_attr(self, name, index):
         # attr cache
-        assert type(name) is str    # utf8-encoded
+        assert isinstance(name, str)    # utf8-encoded
         space = self.space
         cache = space.fromcache(MapAttrCache)
         SHIFT2 = r_uint.BITS - space.config.objspace.std.methodcachesizeexp

From pypy.commits at gmail.com  Sun Aug 21 04:37:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 01:37:39 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: I'll do os.scandir() next
Message-ID: <57b96853.e129c20a.f2ff2.3fe3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5677:6aee284a8009
Date: 2016-08-21 10:37 +0200
http://bitbucket.org/pypy/extradoc/changeset/6aee284a8009/

Log:	I'll do os.scandir() next

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -11,6 +11,8 @@
   We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
   Matti's opinion on that would be great!
 
+* os.scandir() (arigo)
+
 
 Finished
 --------

From pypy.commits at gmail.com  Sun Aug 21 05:48:47 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 21 Aug 2016 02:48:47 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: translation fixes
Message-ID: <57b978ff.482cc20a.97c42.5fcf@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86371:e97ba225e392
Date: 2016-08-21 06:39 +1200
http://bitbucket.org/pypy/pypy/changeset/e97ba225e392/

Log:	translation fixes

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -208,7 +208,8 @@
     def buffer_w(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self, flags)
+            w_result = space.get_and_call_function(w_impl, self, 
+                                        space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
@@ -217,7 +218,7 @@
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
-                                                space.BUF_FULL_RO)
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
@@ -226,7 +227,7 @@
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
-                                                space.BUF_FULL)
+                                        space.newint(space.BUF_FULL))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
@@ -235,7 +236,7 @@
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
-                                                space.BUF_FULL_RO)
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -348,10 +348,7 @@
     with lltype.scoped_alloc(Py_buffer) as pybuf:
         _flags = 0
         if space.len_w(w_args) > 0:
-            _flags = space.listview(w_args)[0]
-        if not isinstance(_flags, int):
-            raise oefmt(space.w_TypeError, 
-                        "non-int flags passed to getbufferproc")
+            _flags = space.int_w(space.listview(w_args)[0])
         flags = rffi.cast(rffi.INT_real,_flags)
         size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
         if widen(size) < 0:

From pypy.commits at gmail.com  Sun Aug 21 05:48:49 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 21 Aug 2016 02:48:49 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: test,
 fix missing attributes
Message-ID: <57b97901.031dc20a.34dd5.6325@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86372:75a113a08fd8
Date: 2016-08-21 14:07 +1000
http://bitbucket.org/pypy/pypy/changeset/75a113a08fd8/

Log:	test, fix missing attributes

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -310,8 +310,12 @@
         self.format = format
         if not shape:
             self.shape = [size]
+        else:
+            self.shape = shape
         if not strides:
             self.strides = [1]
+        else:
+            self.strides = strides
         self.ndim = ndim 
         self.itemsize = itemsize
         self.readonly = readonly
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -25,6 +25,7 @@
         arr = module.PyMyArray(10)
         y = memoryview(arr)
         assert y.format == 'i'
+        assert y.shape == (10,)
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)

From pypy.commits at gmail.com  Sun Aug 21 07:10:22 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 21 Aug 2016 04:10:22 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: merge default into branch
Message-ID: <57b98c1e.03121c0a.636a9.9d58@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86373:6cb76ef9492b
Date: 2016-08-21 21:08 +1000
http://bitbucket.org/pypy/pypy/changeset/6cb76ef9492b/

Log:	merge default into branch

diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -356,6 +356,11 @@
 that a C-level traceback is usually of no help at all in PyPy.
 Debugging PyPy can be annoying.
 
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
 In more details:
 
 * First, please give the exact PyPy version, and the OS.
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -387,7 +387,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -406,8 +407,11 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            assert depth >= 0
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1707,6 +1707,23 @@
                 "Python int too large for C unsigned short")
         return value
 
+    def c_uid_t_w(self, w_obj):
+        # xxx assumes that uid_t and gid_t are a C unsigned int.
+        # Equivalent to space.c_uint_w(), with the exception that
+        # it also accepts -1 and converts that to UINT_MAX, which
+        # is (uid_t)-1.  And values smaller than -1 raise
+        # OverflowError, not ValueError.
+        try:
+            return self.c_uint_w(w_obj)
+        except OperationError as e:
+            if e.match(self, self.w_ValueError):
+                # ValueError: cannot convert negative integer to unsigned
+                if self.int_w(w_obj) == -1:
+                    return UINT_MAX
+                raise oefmt(self.w_OverflowError,
+                            "user/group id smaller than minimum (-1)")
+            raise
+
     def truncatedint_w(self, w_obj, allow_conversion=True):
         # Like space.gateway_int_w(), but return the integer truncated
         # instead of raising OverflowError.  For obscure cases only.
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -167,6 +167,9 @@
     def visit_c_ushort(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_c_uid_t(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_truncatedint_w(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -294,6 +297,9 @@
     def visit_c_ushort(self, typ):
         self.run_args.append("space.c_ushort_w(%s)" % (self.scopenext(),))
 
+    def visit_c_uid_t(self, typ):
+        self.run_args.append("space.c_uid_t_w(%s)" % (self.scopenext(),))
+
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
@@ -440,6 +446,9 @@
     def visit_c_ushort(self, typ):
         self.unwrap.append("space.c_ushort_w(%s)" % (self.nextarg(),))
 
+    def visit_c_uid_t(self, typ):
+        self.unwrap.append("space.c_uid_t_w(%s)" % (self.nextarg(),))
+
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
@@ -587,7 +596,10 @@
 
         # First extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(func.func_code)
+        sig = pycode.cpython_code_signature(func.func_code)
+        argnames = sig.argnames
+        varargname = sig.varargname
+        kwargname = sig.kwargname
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -611,7 +623,9 @@
         app_sig = SignatureBuilder(func)
 
         UnwrapSpec_Check(orig_sig).apply_over(unwrap_spec, app_sig)
-        self.sig = argnames, varargname, kwargname = app_sig.signature()
+        self.sig = app_sig.signature()
+        argnames = self.sig.argnames
+        varargname = self.sig.varargname
 
         self.minargs = len(argnames)
         if varargname:
@@ -942,7 +956,7 @@
                 defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
             UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig[0])
+            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
             if defs_w:
                 alldefs_w[-len(defs_w):] = defs_w
             code = self._code
@@ -959,7 +973,7 @@
                     assert isinstance(w_default, W_Root)
                     assert argname.startswith('w_')
                     argname = argname[2:]
-                    j = self._code.sig[0].index(argname)
+                    j = self._code.sig.argnames.index(argname)
                     assert alldefs_w[j] in (UNDEFINED, None)
                     alldefs_w[j] = w_default
             first_defined = 0
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -37,7 +37,7 @@
 
 # cpython_code_signature helper
 def cpython_code_signature(code):
-    "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
+    """Return a Signature instance."""
     argcount = code.co_argcount
     varnames = code.co_varnames
     assert argcount >= 0     # annotator hint
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -55,18 +55,3 @@
         if not isinstance(other, Signature):
             return NotImplemented
         return not self == other
-
-
-    # make it look tuply for its use in the annotator
-
-    def __len__(self):
-        return 3
-
-    def __getitem__(self, i):
-        if i == 0:
-            return self.argnames
-        if i == 1:
-            return self.varargname
-        if i == 2:
-            return self.kwargname
-        raise IndexError
\ No newline at end of file
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -46,13 +46,6 @@
         assert sig.find_argname("c") == 2
         assert sig.find_argname("d") == -1
 
-    def test_tuply(self):
-        sig = Signature(["a", "b", "c"], "d", "e")
-        x, y, z = sig
-        assert x == ["a", "b", "c"]
-        assert y == "d"
-        assert z == "e"
-
 class dummy_wrapped_dict(dict):
     def __nonzero__(self):
         raise NotImplementedError
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -203,7 +203,8 @@
     name = func.__name__
     extra = ', '.join(extraargs)
     from pypy.interpreter import pycode
-    argnames, _, _ = pycode.cpython_code_signature(func.func_code)
+    sig = pycode.cpython_code_signature(func.func_code)
+    argnames = sig.argnames
     if use_closure:
         if argnames[1] == 'space':
             args = "closure, space, obj"
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -310,11 +310,15 @@
                             self.ctype.name, ct.name)
             #
             itemsize = ct.ctitem.size
-            if itemsize <= 0:
-                itemsize = 1
             with self as ptr1, w_other as ptr2:
                 diff = (rffi.cast(lltype.Signed, ptr1) -
-                        rffi.cast(lltype.Signed, ptr2)) // itemsize
+                        rffi.cast(lltype.Signed, ptr2))
+            if itemsize > 1:
+                if diff % itemsize:
+                    raise oefmt(space.w_ValueError,
+                        "pointer subtraction: the distance between the two "
+                        "pointers is not a multiple of the item size")
+                diff //= itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -253,6 +253,7 @@
     sandboxsafe=True)
 
 # split here for JIT backends that don't support floats/longlongs/etc.
+ at jit.dont_look_inside
 def is_nonnull_longdouble(cdata):
     return _is_nonnull_longdouble(read_raw_longdouble_data(cdata))
 def is_nonnull_float(cdata, size):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -576,6 +576,19 @@
     e = py.test.raises(TypeError, "q - a")
     assert str(e.value) == "cannot subtract cdata 'short *' and cdata 'int *'"
 
+def test_ptr_sub_unaligned():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    a = cast(BIntPtr, 1240)
+    for bi in range(1430, 1438):
+        b = cast(BIntPtr, bi)
+        if ((bi - 1240) % size_of_int()) == 0:
+            assert b - a == (bi - 1240) // size_of_int()
+            assert a - b == (1240 - bi) // size_of_int()
+        else:
+            py.test.raises(ValueError, "b - a")
+            py.test.raises(ValueError, "a - b")
+
 def test_cast_primitive_from_cdata():
     p = new_primitive_type("int")
     n = cast(p, cast(p, -42))
diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,10 +69,13 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
+        cls.w_fix_path = space.appexec([space.wrap(str(tmpdir))], """(path):
+            def fix_path(ignored=None):
+                import _cffi_backend     # force it to be initialized
+                import sys
+                if path not in sys.path:
+                    sys.path.insert(0, path)
+            return fix_path
         """)
 
     def teardown_method(self, meth):
@@ -86,17 +89,20 @@
 
 
     def test_constant_1(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
@@ -104,6 +110,7 @@
 
     def test_dlclose(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
@@ -115,17 +122,20 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
@@ -134,12 +144,14 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
+        self.fix_path()
         from re_py_subsrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
@@ -153,6 +165,7 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
@@ -163,24 +176,28 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == "hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -10,10 +10,10 @@
 
     def setup_class(cls):
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__jitlog.1')))
-        cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
-        cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
+        cls.w_mark_header = cls.space.newbytes(jl.MARK_JITLOG_HEADER)
+        cls.w_version = cls.space.newbytes(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
-        cls.w_machine = cls.space.wrap(platform.machine())
+        cls.w_machine = cls.space.newbytes(platform.machine())
         cls.w_resops = cls.space.newdict()
         space = cls.space
         for key, value in opname.items():
@@ -48,5 +48,3 @@
                 assert opnum in self.resops
                 # the name must equal
                 assert self.resops[opnum] == opname
-
-
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
@@ -268,22 +268,26 @@
     min = 0;
     max = haystacksize;
 
-    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
+    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
         if (value < haystack[pos].uniseq) {
-            if (max == pos) break;
-            else max = pos;
+            if (max != pos) {
+                max = pos;
+                continue;
+            }
         }
         else if (value > haystack[pos].uniseq) {
-            if (min == pos) break;
-            else min = pos;
+            if (min != pos) {
+                min = pos;
+                continue;
+            }
         }
-        else
-            break;
+        break;
+    }
 
-        if (value == haystack[pos].uniseq)
-            return haystack[pos].code;
-        else
-            return DBCINV;
+    if (value == haystack[pos].uniseq) {
+        return haystack[pos].code;
+    }
+    return DBCINV;
 }
 #endif
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,7 +11,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
-from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite, 
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite,
         c_fdopen, c_fileno,
         c_fopen)# for tests
 from rpython.translator import cdir
@@ -259,14 +259,14 @@
 
         # extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(callable.func_code)
+        sig = pycode.cpython_code_signature(callable.func_code)
+        assert sig.argnames[0] == 'space'
+        self.argnames = sig.argnames[1:]
+        if gil == 'pygilstate_ensure':
+            assert self.argnames[-1] == 'previous_state'
+            del self.argnames[-1]
+        assert len(self.argnames) == len(self.argtypes)
 
-        assert argnames[0] == 'space'
-        if gil == 'pygilstate_ensure':
-            assert argnames[-1] == 'previous_state'
-            del argnames[-1]
-        self.argnames = argnames[1:]
-        assert len(self.argnames) == len(self.argtypes)
         self.gil = gil
         self.result_borrowed = result_borrowed
         self.result_is_ll = result_is_ll
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -4,7 +4,7 @@
 from rpython.rlib import rposix, rposix_stat
 from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_longlong, intmask
+from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint
 from rpython.rlib.unroll import unrolling_iterable
 
 from pypy.interpreter.gateway import unwrap_spec
@@ -20,20 +20,21 @@
 
 c_int = "c_int"
 
-# CPython 2.7 semantics are too messy to follow exactly,
-# e.g. setuid(-2) works on 32-bit but not on 64-bit.  As a result,
-# we decided to just accept any 'int', i.e. any C signed long, and
-# check that they are in range(-2**31, 2**32).  In other words, we
-# accept any number that is either a signed or an unsigned C int.
-c_uid_t = int
-c_gid_t = int
-if sys.maxint == 2147483647:
-    def check_uid_range(space, num):
-        pass
-else:
-    def check_uid_range(space, num):
-        if num < -(1 << 31) or num >= (1 << 32):
-            raise oefmt(space.w_OverflowError, "integer out of range")
+# CPython 2.7 semantics used to be too messy, differing on 32-bit vs
+# 64-bit, but this was cleaned up in recent 2.7.x.  Now, any function
+# taking a uid_t or gid_t accepts numbers in range(-1, 2**32) as an
+# r_uint, with -1 being equivalent to 2**32-1.  Any function that
+# returns a uid_t or gid_t returns either an int or a long, depending
+# on whether it fits or not, but always positive.
+c_uid_t = 'c_uid_t'
+c_gid_t = 'c_uid_t'
+
+def wrap_uid(space, uid):
+    if uid <= r_uint(sys.maxint):
+        return space.wrap(intmask(uid))
+    else:
+        return space.wrap(uid)     # an unsigned number
+wrap_gid = wrap_uid
 
 def fsencode_w(space, w_obj):
     if space.isinstance_w(w_obj, space.w_unicode):
@@ -912,7 +913,7 @@
 
     Return the current process's user id.
     """
-    return space.wrap(os.getuid())
+    return wrap_uid(space, os.getuid())
 
 @unwrap_spec(arg=c_uid_t)
 def setuid(space, arg):
@@ -920,12 +921,10 @@
 
     Set the current process's user id.
     """
-    check_uid_range(space, arg)
     try:
         os.setuid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_uid_t)
 def seteuid(space, arg):
@@ -933,12 +932,10 @@
 
     Set the current process's effective user id.
     """
-    check_uid_range(space, arg)
     try:
         os.seteuid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_gid_t)
 def setgid(space, arg):
@@ -946,12 +943,10 @@
 
     Set the current process's group id.
     """
-    check_uid_range(space, arg)
     try:
         os.setgid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(arg=c_gid_t)
 def setegid(space, arg):
@@ -959,12 +954,10 @@
 
     Set the current process's effective group id.
     """
-    check_uid_range(space, arg)
     try:
         os.setegid(arg)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(path='str0')
 def chroot(space, path):
@@ -983,21 +976,21 @@
 
     Return the current process's group id.
     """
-    return space.wrap(os.getgid())
+    return wrap_gid(space, os.getgid())
 
 def getegid(space):
     """ getegid() -> gid
 
     Return the current process's effective group id.
     """
-    return space.wrap(os.getegid())
+    return wrap_gid(space, os.getegid())
 
 def geteuid(space):
     """ geteuid() -> euid
 
     Return the current process's effective user id.
     """
-    return space.wrap(os.geteuid())
+    return wrap_uid(space, os.geteuid())
 
 def getgroups(space):
     """ getgroups() -> list of group IDs
@@ -1008,7 +1001,7 @@
         list = os.getgroups()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newlist([space.wrap(e) for e in list])
+    return space.newlist([wrap_gid(space, e) for e in list])
 
 def setgroups(space, w_list):
     """ setgroups(list)
@@ -1017,9 +1010,7 @@
     """
     list = []
     for w_gid in space.unpackiterable(w_list):
-        gid = space.int_w(w_gid)
-        check_uid_range(space, gid)
-        list.append(gid)
+        list.append(space.c_uid_t_w(w_gid))
     try:
         os.setgroups(list[:])
     except OSError as e:
@@ -1093,13 +1084,10 @@
 
     Set the current process's real and effective user ids.
     """
-    check_uid_range(space, ruid)
-    check_uid_range(space, euid)
     try:
         os.setreuid(ruid, euid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(rgid=c_gid_t, egid=c_gid_t)
 def setregid(space, rgid, egid):
@@ -1107,13 +1095,10 @@
 
     Set the current process's real and effective group ids.
     """
-    check_uid_range(space, rgid)
-    check_uid_range(space, egid)
     try:
         os.setregid(rgid, egid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.w_None
 
 @unwrap_spec(pid=c_int)
 def getsid(space, pid):
@@ -1150,7 +1135,7 @@
         raise wrap_oserror(space, e)
     return space.wrap(pgid)
 
- at unwrap_spec(fd=c_int, pgid=c_gid_t)
+ at unwrap_spec(fd=c_int, pgid=c_int)
 def tcsetpgrp(space, fd, pgid):
     """ tcsetpgrp(fd, pgid)
 
@@ -1170,9 +1155,9 @@
         (ruid, euid, suid) = os.getresuid()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(ruid),
-                           space.wrap(euid),
-                           space.wrap(suid)])
+    return space.newtuple([wrap_uid(space, ruid),
+                           wrap_uid(space, euid),
+                           wrap_uid(space, suid)])
 
 def getresgid(space):
     """ getresgid() -> (rgid, egid, sgid)
@@ -1183,9 +1168,9 @@
         (rgid, egid, sgid) = os.getresgid()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(rgid),
-                           space.wrap(egid),
-                           space.wrap(sgid)])
+    return space.newtuple([wrap_gid(space, rgid),
+                           wrap_gid(space, egid),
+                           wrap_gid(space, sgid)])
 
 @unwrap_spec(ruid=c_uid_t, euid=c_uid_t, suid=c_uid_t)
 def setresuid(space, ruid, euid, suid):
@@ -1284,8 +1269,6 @@
 @unwrap_spec(path='str0', uid=c_uid_t, gid=c_gid_t)
 def chown(space, path, uid, gid):
     """Change the owner and group id of path to the numeric uid and gid."""
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.chown(path, uid, gid)
     except OSError as e:
@@ -1295,8 +1278,6 @@
 def lchown(space, path, uid, gid):
     """Change the owner and group id of path to the numeric uid and gid.
 This function will not follow symbolic links."""
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.lchown(path, uid, gid)
     except OSError as e:
@@ -1307,8 +1288,6 @@
     """Change the owner and group id of the file given by file descriptor
 fd to the numeric uid and gid."""
     fd = space.c_filedescriptor_w(w_fd)
-    check_uid_range(space, uid)
-    check_uid_range(space, gid)
     try:
         os.fchown(fd, uid, gid)
     except OSError as e:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -619,10 +619,12 @@
             assert os.geteuid() == self.geteuid
 
     if hasattr(os, 'setuid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setuid_error(self):
             os = self.posix
-            raises(OverflowError, os.setuid, -2**31-1)
+            raises(OverflowError, os.setuid, -2)
             raises(OverflowError, os.setuid, 2**32)
+            raises(OSError, os.setuid, -1)
 
     if hasattr(os, 'getgid'):
         def test_os_getgid(self):
@@ -665,10 +667,14 @@
             raises(OSError, os.getpgid, 1234567)
 
     if hasattr(os, 'setgid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setgid_error(self):
             os = self.posix
-            raises(OverflowError, os.setgid, -2**31-1)
+            raises(OverflowError, os.setgid, -2)
             raises(OverflowError, os.setgid, 2**32)
+            raises(OSError, os.setgid, -1)
+            raises(OSError, os.setgid, -1L)
+            raises(OSError, os.setgid, 2**32-1)
 
     if hasattr(os, 'getsid'):
         def test_os_getsid(self):
diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -45,5 +45,8 @@
 
 def test_setrlimit():
     # minimal "does not crash" test
-    x = resource.getrlimit(resource.RLIMIT_CPU)
-    resource.setrlimit(resource.RLIMIT_CPU, x)
+    x, y = resource.getrlimit(resource.RLIMIT_CPU)
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))
+    x += 0.2
+    y += 0.3
+    resource.setrlimit(resource.RLIMIT_CPU, (x, y))    # truncated to ints
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -221,7 +221,6 @@
         stack_index = 0
         while True:
             current = self
-            number_to_readd = 0
             number_to_readd, attr = self._find_branch_to_move_into(name, index)
             # we found the attributes further up, need to save the
             # previous values of the attributes we passed
diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py
--- a/pypy/tool/pytest/genreportdata.py
+++ b/pypy/tool/pytest/genreportdata.py
@@ -17,7 +17,7 @@
             resultwc = py.path.svnwc(testresultdir)
             print "updating", resultwc
             resultwc.update()
-        except KeyboardInterrupt as RuntimeError:
+        except (KeyboardInterrupt, RuntimeError):
             raise
         except Exception as e: #py.process.ExecutionFailed,e:
             print >> sys.stderr, "Warning: ",e #Subversion update failed"
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
+cffi>=1.4.0
+
 # hypothesis is used for test generation on untranslated tests
 hypothesis
 enum34>=1.1.2
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1051,8 +1051,9 @@
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     #
+    jd_name = jitdriver_sd.jitdriver.name
     metainterp_sd.jitlog.start_new_trace(metainterp_sd,
-            faildescr=resumekey, entry_bridge=False)
+            faildescr=resumekey, entry_bridge=False, jd_name=jd_name)
     #
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -98,6 +98,7 @@
     def log_abort_loop(self, trace, memo=None):
         debug_start("jit-abort-log")
         if not have_debug_prints():
+            debug_stop("jit-abort-log")
             return
         inputargs, operations = self._unpack_trace(trace)
         logops = self._log_operations(inputargs, operations, ops_offset=None,
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -64,6 +64,10 @@
     testself.all_graphs = graphs
     result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0]
 
+
+    class FakeJitDriver:
+        name = 'fakejitdriver'
+
     class FakeJitDriverSD:
         num_green_args = 0
         portal_graph = graphs[0]
@@ -72,6 +76,7 @@
         result_type = result_kind
         portal_runner_ptr = "???"
         vec = False
+        jitdriver = FakeJitDriver()
 
     stats = history.Stats(None)
     cpu = CPUClass(rtyper, stats, None, False)
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 1
+JITLOG_VERSION = 2
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -331,7 +331,7 @@
     def finish(self):
         jitlog_teardown()
 
-    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False):
+    def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False, jd_name=""):
         # even if the logger is not enabled, increment the trace id
         self.trace_id += 1
         if not jitlog_enabled():
@@ -345,6 +345,7 @@
         else:
             content.append(encode_str('loop'))
             content.append(encode_le_addr(int(entry_bridge)))
+        content.append(encode_str(jd_name))
         self._write_marked(MARK_START_TRACE, ''.join(content))
 
     def trace_aborted(self):
@@ -508,7 +509,9 @@
         """ an operation is written as follows:
             <marker> <opid (16 bit)> \
                      <len (32 bit)> \
-                     <res_val>,<arg_0>,...,<arg_n>,<descr>
+                     <res_val>,<arg_0>,...,<arg_n> \
+                     <descr>
+                     <failarg_0>,...<failarg_n>
             The marker indicates if the last argument is
             a descr or a normal argument.
         """
@@ -517,16 +520,21 @@
         le_opnum = encode_le_16bit(op.getopnum())
         str_res = self.var_to_str(op)
         line = ','.join([str_res] + str_args)
+        failargslist = op.getfailargs()
+        failargs = ''
+        if failargslist:
+            failargs = ','.join([self.var_to_str(farg) for farg in failargslist])
+        #
         if descr:
             descr_str = descr.repr_of_descr()
             line = line + ',' + descr_str
             string = encode_str(line)
             descr_number = compute_unique_id(descr)
             le_descr_number = encode_le_addr(descr_number)
-            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number
+            return MARK_RESOP_DESCR, le_opnum + string + le_descr_number + encode_str(failargs)
         else:
             string = encode_str(line)
-            return MARK_RESOP, le_opnum + string
+            return MARK_RESOP, le_opnum + string + encode_str(failargs)
 
 
     def write_core_dump(self, operations, i, op, ops_offset):
@@ -578,6 +586,8 @@
         return ''.join(dump)
 
     def var_to_str(self, arg):
+        if arg is None:
+            return '-'
         try:
             mv = self.memo[arg]
         except KeyError:
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -48,7 +48,7 @@
         file.ensure()
         fd = file.open('wb')
         jl.jitlog_init(fd.fileno())
-        logger.start_new_trace(self.make_metainterp_sd())
+        logger.start_new_trace(self.make_metainterp_sd(), jd_name='jdname')
         log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
         op = ResOperation(rop.DEBUG_MERGE_POINT, [ConstInt(0), ConstInt(0), ConstInt(0)])
         log_trace.write([], [op])
@@ -58,6 +58,7 @@
         is_32bit = chr(sys.maxint == 2**31-1)
         assert binary == (jl.MARK_START_TRACE) + jl.encode_le_addr(1) + \
                          jl.encode_str('loop') + jl.encode_le_addr(0) + \
+                         jl.encode_str('jdname') + \
                          (jl.MARK_TRACE) + jl.encode_le_addr(1) + \
                          (jl.MARK_INPUT_ARGS) + jl.encode_str('') + \
                          (jl.MARK_INIT_MERGE_POINT) + b'\x05\x00\x01s\x00i\x08s\x00i\x10s' + \
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -96,12 +96,15 @@
             return 0;
         }
     ''',]
+    post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);']
 else:
     separate_module_sources = []
+    post_include_bits = []
     includes=['errno.h','stdio.h']
 errno_eci = ExternalCompilationInfo(
     includes=includes,
     separate_module_sources=separate_module_sources,
+    post_include_bits=post_include_bits,
 )
 
 # Direct getters/setters, don't use directly!
@@ -251,6 +254,8 @@
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
     if not _WIN32:
+        UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
+        GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
         CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
 
         TMS = rffi_platform.Struct(
@@ -1450,32 +1455,33 @@
 def setpgid(pid, gid):
     handle_posix_error('setpgid', c_setpgid(pid, gid))
 
-PID_GROUPS_T = rffi.CArrayPtr(rffi.PID_T)
-c_getgroups = external('getgroups', [rffi.INT, PID_GROUPS_T], rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setgroups = external('setgroups', [rffi.SIZE_T, PID_GROUPS_T], rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_initgroups = external('initgroups', [rffi.CCHARP, rffi.PID_T], rffi.INT,
-                        save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    GID_GROUPS_T = rffi.CArrayPtr(GID_T)
+    c_getgroups = external('getgroups', [rffi.INT, GID_GROUPS_T], rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setgroups = external('setgroups', [rffi.SIZE_T, GID_GROUPS_T], rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_initgroups = external('initgroups', [rffi.CCHARP, GID_T], rffi.INT,
+                            save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('getgroups')
 def getgroups():
     n = handle_posix_error('getgroups',
-                           c_getgroups(0, lltype.nullptr(PID_GROUPS_T.TO)))
-    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+                           c_getgroups(0, lltype.nullptr(GID_GROUPS_T.TO)))
+    groups = lltype.malloc(GID_GROUPS_T.TO, n, flavor='raw')
     try:
         n = handle_posix_error('getgroups', c_getgroups(n, groups))
-        return [widen(groups[i]) for i in range(n)]
+        return [widen_gid(groups[i]) for i in range(n)]
     finally:
         lltype.free(groups, flavor='raw')
 
 @replace_os_function('setgroups')
 def setgroups(gids):
     n = len(gids)
-    groups = lltype.malloc(PID_GROUPS_T.TO, n, flavor='raw')
+    groups = lltype.malloc(GID_GROUPS_T.TO, n, flavor='raw')
     try:
         for i in range(n):
-            groups[i] = rffi.cast(rffi.PID_T, gids[i])
+            groups[i] = rffi.cast(GID_T, gids[i])
         handle_posix_error('setgroups', c_setgroups(n, groups))
     finally:
         lltype.free(groups, flavor='raw')
@@ -1526,104 +1532,115 @@
 
 #___________________________________________________________________
 
-c_getuid = external('getuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_geteuid = external('geteuid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_setuid = external('setuid', [rffi.INT], rffi.INT,
-                    save_err=rffi.RFFI_SAVE_ERRNO)
-c_seteuid = external('seteuid', [rffi.INT], rffi.INT,
-                     save_err=rffi.RFFI_SAVE_ERRNO)
-c_getgid = external('getgid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_getegid = external('getegid', [], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
-c_setgid = external('setgid', [rffi.INT], rffi.INT,
-                    save_err=rffi.RFFI_SAVE_ERRNO)
-c_setegid = external('setegid', [rffi.INT], rffi.INT,
-                     save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    c_getuid = external('getuid', [], UID_T)
+    c_geteuid = external('geteuid', [], UID_T)
+    c_setuid = external('setuid', [UID_T], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+    c_seteuid = external('seteuid', [UID_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    c_getgid = external('getgid', [], GID_T)
+    c_getegid = external('getegid', [], GID_T)
+    c_setgid = external('setgid', [GID_T], rffi.INT,
+                        save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setegid = external('setegid', [GID_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 
- at replace_os_function('getuid')
-def getuid():
-    return handle_posix_error('getuid', c_getuid())
+    def widen_uid(x):
+        return rffi.cast(lltype.Unsigned, x)
+    widen_gid = widen_uid
 
- at replace_os_function('geteuid')
-def geteuid():
-    return handle_posix_error('geteuid', c_geteuid())
+    # NOTE: the resulting type of functions that return a uid/gid is
+    # always Unsigned.  The argument type of functions that take a
+    # uid/gid should also be Unsigned.
 
- at replace_os_function('setuid')
-def setuid(uid):
-    handle_posix_error('setuid', c_setuid(uid))
+    @replace_os_function('getuid')
+    def getuid():
+        return widen_uid(c_getuid())
 
- at replace_os_function('seteuid')
-def seteuid(uid):
-    handle_posix_error('seteuid', c_seteuid(uid))
+    @replace_os_function('geteuid')
+    def geteuid():
+        return widen_uid(c_geteuid())
 
- at replace_os_function('getgid')
-def getgid():
-    return handle_posix_error('getgid', c_getgid())
+    @replace_os_function('setuid')
+    def setuid(uid):
+        handle_posix_error('setuid', c_setuid(uid))
 
- at replace_os_function('getegid')
-def getegid():
-    return handle_posix_error('getegid', c_getegid())
+    @replace_os_function('seteuid')
+    def seteuid(uid):
+        handle_posix_error('seteuid', c_seteuid(uid))
 
- at replace_os_function('setgid')
-def setgid(gid):
-    handle_posix_error('setgid', c_setgid(gid))
+    @replace_os_function('getgid')
+    def getgid():
+        return widen_gid(c_getgid())
 
- at replace_os_function('setegid')
-def setegid(gid):
-    handle_posix_error('setegid', c_setegid(gid))
+    @replace_os_function('getegid')
+    def getegid():
+        return widen_gid(c_getegid())
 
-c_setreuid = external('setreuid', [rffi.INT, rffi.INT], rffi.INT,
-                      save_err=rffi.RFFI_SAVE_ERRNO)
-c_setregid = external('setregid', [rffi.INT, rffi.INT], rffi.INT,
-                      save_err=rffi.RFFI_SAVE_ERRNO)
+    @replace_os_function('setgid')
+    def setgid(gid):
+        handle_posix_error('setgid', c_setgid(gid))
 
- at replace_os_function('setreuid')
-def setreuid(ruid, euid):
-    handle_posix_error('setreuid', c_setreuid(ruid, euid))
+    @replace_os_function('setegid')
+    def setegid(gid):
+        handle_posix_error('setegid', c_setegid(gid))
 
- at replace_os_function('setregid')
-def setregid(rgid, egid):
-    handle_posix_error('setregid', c_setregid(rgid, egid))
+    c_setreuid = external('setreuid', [UID_T, UID_T], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setregid = external('setregid', [GID_T, GID_T], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
-c_getresuid = external('getresuid', [rffi.INTP] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_getresgid = external('getresgid', [rffi.INTP] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setresuid = external('setresuid', [rffi.INT] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
-c_setresgid = external('setresgid', [rffi.INT] * 3, rffi.INT,
-                       save_err=rffi.RFFI_SAVE_ERRNO)
+    @replace_os_function('setreuid')
+    def setreuid(ruid, euid):
+        handle_posix_error('setreuid', c_setreuid(ruid, euid))
 
- at replace_os_function('getresuid')
-def getresuid():
-    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-    try:
-        handle_posix_error('getresuid',
-                           c_getresuid(rffi.ptradd(out, 0),
-                                       rffi.ptradd(out, 1),
-                                       rffi.ptradd(out, 2)))
-        return (widen(out[0]), widen(out[1]), widen(out[2]))
-    finally:
-        lltype.free(out, flavor='raw')
+    @replace_os_function('setregid')
+    def setregid(rgid, egid):
+        handle_posix_error('setregid', c_setregid(rgid, egid))
 
- at replace_os_function('getresgid')
-def getresgid():
-    out = lltype.malloc(rffi.INTP.TO, 3, flavor='raw')
-    try:
-        handle_posix_error('getresgid',
-                           c_getresgid(rffi.ptradd(out, 0),
-                                       rffi.ptradd(out, 1),
-                                       rffi.ptradd(out, 2)))
-        return (widen(out[0]), widen(out[1]), widen(out[2]))
-    finally:
-        lltype.free(out, flavor='raw')
+    UID_T_P = lltype.Ptr(lltype.Array(UID_T, hints={'nolength': True}))
+    GID_T_P = lltype.Ptr(lltype.Array(GID_T, hints={'nolength': True}))
+    c_getresuid = external('getresuid', [UID_T_P] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_getresgid = external('getresgid', [GID_T_P] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setresuid = external('setresuid', [UID_T] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setresgid = external('setresgid', [GID_T] * 3, rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
 
- at replace_os_function('setresuid')
-def setresuid(ruid, euid, suid):
-    handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+    @replace_os_function('getresuid')
+    def getresuid():
+        out = lltype.malloc(UID_T_P.TO, 3, flavor='raw')
+        try:
+            handle_posix_error('getresuid',
+                               c_getresuid(rffi.ptradd(out, 0),
+                                           rffi.ptradd(out, 1),
+                                           rffi.ptradd(out, 2)))
+            return (widen_uid(out[0]), widen_uid(out[1]), widen_uid(out[2]))
+        finally:
+            lltype.free(out, flavor='raw')
 
- at replace_os_function('setresgid')
-def setresgid(rgid, egid, sgid):
-    handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
+    @replace_os_function('getresgid')
+    def getresgid():
+        out = lltype.malloc(GID_T_P.TO, 3, flavor='raw')
+        try:
+            handle_posix_error('getresgid',
+                               c_getresgid(rffi.ptradd(out, 0),
+                                           rffi.ptradd(out, 1),
+                                           rffi.ptradd(out, 2)))
+            return (widen_gid(out[0]), widen_gid(out[1]), widen_gid(out[2]))
+        finally:
+            lltype.free(out, flavor='raw')
+
+    @replace_os_function('setresuid')
+    def setresuid(ruid, euid, suid):
+        handle_posix_error('setresuid', c_setresuid(ruid, euid, suid))
+
+    @replace_os_function('setresgid')
+    def setresgid(rgid, egid, sgid):
+        handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
 
 #___________________________________________________________________
 
@@ -2046,3 +2063,40 @@
     def mknodat(path, mode, device, dir_fd=AT_FDCWD):
         error = c_mknodat(dir_fd, path, mode, device)
         handle_posix_error('mknodat', error)
+
+
+eci_inheritable = eci.merge(ExternalCompilationInfo(
+    separate_module_sources=["""
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable)
+{
+    /* XXX minimal impl. XXX */
+    int request = inheritable ? FIONCLEX : FIOCLEX;
+    return ioctl(fd, request, NULL);
+}
+RPY_EXTERN
+int rpy_get_inheritable(int fd)
+{
+    int flags = fcntl(fd, F_GETFD, 0);
+    if (flags == -1)
+        return -1;
+    return !(flags & FD_CLOEXEC);
+}
+    """],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                             compilation_info=eci_inheritable)
+
+def set_inheritable(fd, inheritable):
+    error = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', error)
+
+def get_inheritable(fd):
+    res = c_get_inheritable(fd)
+    res = handle_posix_error('get_inheritable', res)
+    return res != 0
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -572,3 +572,12 @@
         os.close(dirfd)
     assert tmpdir.join('file').check(exists=False)
     assert tmpdir.join('file2').check(exists=True)
+
+def test_set_inheritable():
+    fd1, fd2 = os.pipe()
+    rposix.set_inheritable(fd1, True)
+    assert rposix.get_inheritable(fd1) == True
+    rposix.set_inheritable(fd1, False)
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)

From pypy.commits at gmail.com  Sun Aug 21 14:23:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 11:23:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Start writing an RPython interface that
 should allow us to portably
Message-ID: <57b9f196.411d1c0a.3ba92.280c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86374:e139e4ad76df
Date: 2016-08-21 18:58 +0200
http://bitbucket.org/pypy/pypy/changeset/e139e4ad76df/

Log:	Start writing an RPython interface that should allow us to portably
	implement os.scandir() in py3.5

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -250,6 +250,7 @@
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
     HAVE_UTIMES = rffi_platform.Has('utimes')
+    HAVE_D_TYPE = rffi_platform.Has('DT_UNKNOWN')
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
@@ -603,11 +604,17 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))])
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
+        if HAVE_D_TYPE:
+            DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
+            DT_REG     = rffi_platform.ConstantInteger('DT_REG')
+            DT_DIR     = rffi_platform.ConstantInteger('DT_DIR')
+            DT_LNK     = rffi_platform.ConstantInteger('DT_LNK')
 
     DIRP = rffi.COpaquePtr('DIR')
-    config = rffi_platform.configure(CConfig)
-    DIRENT = config['DIRENT']
+    dirent_config = rffi_platform.configure(CConfig)
+    DIRENT = dirent_config['DIRENT']
     DIRENTP = lltype.Ptr(DIRENT)
     c_opendir = external('opendir',
         [rffi.CCHARP], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
@@ -617,7 +624,9 @@
     # dirent struct (which depends on defines)
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
-    c_closedir = external('closedir', [DIRP], rffi.INT)
+    c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
+else:
+    dirent_config = {}
 
 def _listdir(dirp):
     result = []
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rposix_scandir.py
@@ -0,0 +1,59 @@
+from rpython.rlib import rposix
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.lltypesystem import rffi
+
+
+ at specialize.argtype(0)
+def opendir(path):
+    path = rposix._as_bytes0(path)
+    return opendir_bytes(path)
+
+def opendir_bytes(path):
+    dirp = rposix.c_opendir(path)
+    if not dirp:
+        raise OSError(rposix.get_saved_errno(), "opendir failed")
+    return dirp
+
+def closedir(dirp):
+    rposix.c_closedir(dirp)
+
+def nextentry(dirp):
+    """Read the next entry and returns an opaque object.
+    Use the methods has_xxx() and get_xxx() to read from that
+    opaque object.  The opaque object is valid until the next
+    time nextentry() or closedir() is called.  This may raise
+    StopIteration, or OSError.  Note that this doesn't filter
+    out the "." and ".." entries.
+    """
+    direntp = rposix.c_readdir(dirp)
+    if direntp:
+        return direntp
+    error = rposix.get_saved_errno()
+    if error:
+        raise OSError(error, "readdir failed")
+    raise StopIteration
+
+def has_name_bytes(direntp):
+    return True
+
+def get_name_bytes(direntp):
+    namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
+    return rffi.charp2str(namep)
+
+DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', None)
+DT_REG = rposix.dirent_config.get('DT_REG', None)
+DT_DIR = rposix.dirent_config.get('DT_DIR', None)
+DT_LNK = rposix.dirent_config.get('DT_LNK', None)
+
+def has_type(direntp):
+    return (DT_UNKNOWN is not None and
+            rffi.getintfield(direntp, 'c_d_type') != DT_UNKNOWN)
+
+def type_is_regular(direntp):
+    return rffi.getintfield(direntp, 'c_d_type') == DT_REG
+
+def type_is_dir(direntp):
+    return rffi.getintfield(direntp, 'c_d_type') == DT_DIR
+
+def type_is_link(direntp):
+    return rffi.getintfield(direntp, 'c_d_type') == DT_LNK
diff --git a/rpython/rlib/test/test_rposix_scandir.py b/rpython/rlib/test/test_rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rposix_scandir.py
@@ -0,0 +1,22 @@
+import sys, os
+import py
+from rpython.rlib import rposix_scandir
+
+
+class TestScanDir(object):
+
+    @py.test.mark.skipif("sys.platform == 'win32'")   # XXX
+    def test_name_bytes(self):
+        scan = rposix_scandir.opendir('/')
+        found = []
+        while True:
+            try:
+                p = rposix_scandir.nextentry(scan)
+            except StopIteration:
+                break
+            assert rposix_scandir.has_name_bytes(p)
+            found.append(rposix_scandir.get_name_bytes(p))
+        rposix_scandir.closedir(scan)
+        found.remove('.')
+        found.remove('..')
+        assert sorted(found) == sorted(os.listdir('/'))

From pypy.commits at gmail.com  Sun Aug 21 14:23:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 11:23:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Returning a choice of bytes or unicodes
Message-ID: <57b9f19d.a699c20a.b61fd.0530@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86378:50bce54b1c9d
Date: 2016-08-21 20:01 +0200
http://bitbucket.org/pypy/pypy/changeset/50bce54b1c9d/

Log:	Returning a choice of bytes or unicodes

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -3,7 +3,7 @@
 from rpython.rtyper.lltypesystem import lltype
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, interp2app
-from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
+from pypy.interpreter.error import OperationError, oefmt, wrap_oserror2
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.baseobjspace import W_Root
 
@@ -15,7 +15,8 @@
     "scandir(path='.') -> iterator of DirEntry objects for given path"
 
     if space.isinstance_w(w_path, space.w_bytes):
-        XXX
+        path_bytes = space.str0_w(w_path)
+        result_is_bytes = True
     else:
         try:
             path_bytes = space.fsencode_w(w_path)
@@ -24,23 +25,27 @@
                 raise
             fd = unwrap_fd(space, w_path, "string, bytes or integer")
             XXXX
+        result_is_bytes = False
 
-        try:
-            dirp = rposix_scandir.opendir(path_bytes)
-        except OSError as e:
-            raise wrap_oserror(space, e)
-        path_prefix = path_bytes
-        if len(path_prefix) > 0 and path_prefix[-1] != '/':
-            path_prefix += '/'
-        w_path_prefix = space.fsdecode(space.newbytes(path_prefix))
-        return W_ScandirIterator(space, dirp, w_path_prefix)
+    try:
+        dirp = rposix_scandir.opendir(path_bytes)
+    except OSError as e:
+        raise wrap_oserror2(space, e, w_path)
+    path_prefix = path_bytes
+    if len(path_prefix) > 0 and path_prefix[-1] != '/':
+        path_prefix += '/'
+    w_path_prefix = space.newbytes(path_prefix)
+    if not result_is_bytes:
+        w_path_prefix = space.fsdecode(w_path_prefix)
+    return W_ScandirIterator(space, dirp, w_path_prefix, result_is_bytes)
 
 
 class W_ScandirIterator(W_Root):
-    def __init__(self, space, dirp, w_path_prefix):
+    def __init__(self, space, dirp, w_path_prefix, result_is_bytes):
         self.space = space
         self.dirp = dirp
         self.w_path_prefix = w_path_prefix
+        self.result_is_bytes = result_is_bytes
 
     @rgc.must_be_light_finalizer
     def __del__(self):
@@ -79,7 +84,7 @@
             if name != '.' and name != '..':
                 break
         #
-        direntry = W_DirEntry(name, self.w_path_prefix)
+        direntry = W_DirEntry(name, self.w_path_prefix, self.result_is_bytes)
         return space.wrap(direntry)
 
 
@@ -95,14 +100,17 @@
     w_name = None
     w_path = None
 
-    def __init__(self, name_bytes, w_path_prefix):
+    def __init__(self, name_bytes, w_path_prefix, result_is_bytes):
         self.name_bytes = name_bytes
         self.w_path_prefix = w_path_prefix
+        self.result_is_bytes = result_is_bytes
 
     def fget_name(self, space):
         w_name = self.w_name
         if w_name is None:
-            w_name = space.fsdecode(space.newbytes(self.name_bytes))
+            w_name = space.newbytes(self.name_bytes)
+            if not self.result_is_bytes:
+                w_name = space.fsdecode(w_name)
             self.w_name = w_name
         return w_name
 

From pypy.commits at gmail.com  Sun Aug 21 14:23:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 11:23:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57b9f198.c997c20a.52bfe.fee7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86375:52ba32002a16
Date: 2016-08-21 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/52ba32002a16/

Log:	hg merge default

diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,21 +69,13 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        cls.w_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
-            from re_python_pysrc import ffi
-            del sys.path[0]
-            return ffi
-        """)
-        cls.w_sub_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
-            from re_py_subsrc import ffi
-            del sys.path[0]
-            return ffi
+        cls.w_fix_path = space.appexec([space.wrap(str(tmpdir))], """(path):
+            def fix_path(ignored=None):
+                import _cffi_backend     # force it to be initialized
+                import sys
+                if path not in sys.path:
+                    sys.path.insert(0, path)
+            return fix_path
         """)
 
     def teardown_method(self, meth):
@@ -97,25 +89,29 @@
 
 
     def test_constant_1(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
         assert type(lib.add42) is _cffi_backend.FFI.CData
 
     def test_dlclose(self):
         import _cffi_backend
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
         e = raises(ffi.error, ffi.dlclose, lib)
@@ -126,18 +122,21 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
             assert p.x == 5
@@ -145,13 +144,15 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
-        ffi = self.sub_ffi
+        self.fix_path()
+        from re_py_subsrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
         assert ffi.integer_const('k2') == 121212
@@ -164,7 +165,8 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
         p = ffi.addressof(lib, 'globalvar42')
@@ -174,25 +176,29 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == b"hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
         assert str(e.value).startswith(
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -593,6 +593,7 @@
             assert os.geteuid() == self.geteuid
 
     if hasattr(os, 'setuid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setuid_error(self):
             os = self.posix
             raises(OverflowError, os.setuid, -2)
@@ -640,6 +641,7 @@
             raises(OSError, os.getpgid, 1234567)
 
     if hasattr(os, 'setgid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setgid_error(self):
             os = self.posix
             raises(OverflowError, os.setgid, -2)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -250,6 +250,7 @@
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
     HAVE_UTIMES = rffi_platform.Has('utimes')
+    HAVE_D_TYPE = rffi_platform.Has('DT_UNKNOWN')
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
@@ -603,11 +604,17 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))])
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
+        if HAVE_D_TYPE:
+            DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
+            DT_REG     = rffi_platform.ConstantInteger('DT_REG')
+            DT_DIR     = rffi_platform.ConstantInteger('DT_DIR')
+            DT_LNK     = rffi_platform.ConstantInteger('DT_LNK')
 
     DIRP = rffi.COpaquePtr('DIR')
-    config = rffi_platform.configure(CConfig)
-    DIRENT = config['DIRENT']
+    dirent_config = rffi_platform.configure(CConfig)
+    DIRENT = dirent_config['DIRENT']
     DIRENTP = lltype.Ptr(DIRENT)
     c_opendir = external('opendir',
         [rffi.CCHARP], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
@@ -617,7 +624,9 @@
     # dirent struct (which depends on defines)
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
-    c_closedir = external('closedir', [DIRP], rffi.INT)
+    c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
+else:
+    dirent_config = {}
 
 def _listdir(dirp):
     result = []
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rposix_scandir.py
@@ -0,0 +1,59 @@
+from rpython.rlib import rposix
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.lltypesystem import rffi
+
+
+ at specialize.argtype(0)
+def opendir(path):
+    path = rposix._as_bytes0(path)
+    return opendir_bytes(path)
+
+def opendir_bytes(path):
+    dirp = rposix.c_opendir(path)
+    if not dirp:
+        raise OSError(rposix.get_saved_errno(), "opendir failed")
+    return dirp
+
+def closedir(dirp):
+    rposix.c_closedir(dirp)
+
+def nextentry(dirp):
+    """Read the next entry and returns an opaque object.
+    Use the methods has_xxx() and get_xxx() to read from that
+    opaque object.  The opaque object is valid until the next
+    time nextentry() or closedir() is called.  This may raise
+    StopIteration, or OSError.  Note that this doesn't filter
+    out the "." and ".." entries.
+    """
+    direntp = rposix.c_readdir(dirp)
+    if direntp:
+        return direntp
+    error = rposix.get_saved_errno()
+    if error:
+        raise OSError(error, "readdir failed")
+    raise StopIteration
+
+def has_name_bytes(direntp):
+    return True
+
+def get_name_bytes(direntp):
+    namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
+    return rffi.charp2str(namep)
+
+DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', None)
+DT_REG = rposix.dirent_config.get('DT_REG', None)
+DT_DIR = rposix.dirent_config.get('DT_DIR', None)
+DT_LNK = rposix.dirent_config.get('DT_LNK', None)
+
+def has_type(direntp):
+    return (DT_UNKNOWN is not None and
+            rffi.getintfield(direntp, 'c_d_type') != DT_UNKNOWN)
+
+def type_is_regular(direntp):
+    return rffi.getintfield(direntp, 'c_d_type') == DT_REG
+
+def type_is_dir(direntp):
+    return rffi.getintfield(direntp, 'c_d_type') == DT_DIR
+
+def type_is_link(direntp):
+    return rffi.getintfield(direntp, 'c_d_type') == DT_LNK
diff --git a/rpython/rlib/test/test_rposix_scandir.py b/rpython/rlib/test/test_rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rposix_scandir.py
@@ -0,0 +1,22 @@
+import sys, os
+import py
+from rpython.rlib import rposix_scandir
+
+
+class TestScanDir(object):
+
+    @py.test.mark.skipif("sys.platform == 'win32'")   # XXX
+    def test_name_bytes(self):
+        scan = rposix_scandir.opendir('/')
+        found = []
+        while True:
+            try:
+                p = rposix_scandir.nextentry(scan)
+            except StopIteration:
+                break
+            assert rposix_scandir.has_name_bytes(p)
+            found.append(rposix_scandir.get_name_bytes(p))
+        rposix_scandir.closedir(scan)
+        found.remove('.')
+        found.remove('..')
+        assert sorted(found) == sorted(os.listdir('/'))

From pypy.commits at gmail.com  Sun Aug 21 14:23:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 11:23:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: No point in making 'name' lazy,
 and it's better if the EncodingError
Message-ID: <57b9f19f.eeb8c20a.7e972.0e78@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86379:7c5ace65de0a
Date: 2016-08-21 20:22 +0200
http://bitbucket.org/pypy/pypy/changeset/7c5ace65de0a/

Log:	No point in making 'name' lazy, and it's better if the EncodingError
	we might get is thrown at the same place as in CPython

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -84,7 +84,11 @@
             if name != '.' and name != '..':
                 break
         #
-        direntry = W_DirEntry(name, self.w_path_prefix, self.result_is_bytes)
+        w_name = space.newbytes(name)
+        result_is_bytes = self.result_is_bytes
+        if not result_is_bytes:
+            w_name = space.fsdecode(w_name)
+        direntry = W_DirEntry(w_name, self.w_path_prefix, result_is_bytes)
         return space.wrap(direntry)
 
 
@@ -97,28 +101,20 @@
 
 
 class W_DirEntry(W_Root):
-    w_name = None
     w_path = None
 
-    def __init__(self, name_bytes, w_path_prefix, result_is_bytes):
-        self.name_bytes = name_bytes
+    def __init__(self, w_name, w_path_prefix, result_is_bytes):
+        self.w_name = w_name
         self.w_path_prefix = w_path_prefix
         self.result_is_bytes = result_is_bytes
 
     def fget_name(self, space):
-        w_name = self.w_name
-        if w_name is None:
-            w_name = space.newbytes(self.name_bytes)
-            if not self.result_is_bytes:
-                w_name = space.fsdecode(w_name)
-            self.w_name = w_name
-        return w_name
+        return self.w_name
 
     def fget_path(self, space):
         w_path = self.w_path
         if w_path is None:
-            w_name = self.fget_name(space)
-            w_path = space.add(self.w_path_prefix, w_name)
+            w_path = space.add(self.w_path_prefix, self.w_name)
             self.w_path = w_path
         return w_path
 

From pypy.commits at gmail.com  Sun Aug 21 14:23:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 11:23:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57b9f19a.2a0dc20a.f915f.0b6c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86376:384bd70a1123
Date: 2016-08-21 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/384bd70a1123/

Log:	hg merge py3k

diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,21 +69,13 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        cls.w_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
-            from re_python_pysrc import ffi
-            del sys.path[0]
-            return ffi
-        """)
-        cls.w_sub_ffi = space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
-            from re_py_subsrc import ffi
-            del sys.path[0]
-            return ffi
+        cls.w_fix_path = space.appexec([space.wrap(str(tmpdir))], """(path):
+            def fix_path(ignored=None):
+                import _cffi_backend     # force it to be initialized
+                import sys
+                if path not in sys.path:
+                    sys.path.insert(0, path)
+            return fix_path
         """)
 
     def teardown_method(self, meth):
@@ -97,25 +89,29 @@
 
 
     def test_constant_1(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
         assert type(lib.add42) is _cffi_backend.FFI.CData
 
     def test_dlclose(self):
         import _cffi_backend
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
         e = raises(ffi.error, ffi.dlclose, lib)
@@ -126,18 +122,21 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
             assert p.x == 5
@@ -145,13 +144,15 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
-        ffi = self.sub_ffi
+        self.fix_path()
+        from re_py_subsrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
         assert ffi.integer_const('k2') == 121212
@@ -164,7 +165,8 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
         p = ffi.addressof(lib, 'globalvar42')
@@ -174,25 +176,29 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == b"hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
-        ffi = self.ffi
+        self.fix_path()
+        from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
         assert str(e.value).startswith(
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -593,6 +593,7 @@
             assert os.geteuid() == self.geteuid
 
     if hasattr(os, 'setuid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setuid_error(self):
             os = self.posix
             raises(OverflowError, os.setuid, -2)
@@ -640,6 +641,7 @@
             raises(OSError, os.getpgid, 1234567)
 
     if hasattr(os, 'setgid'):
+        @py.test.mark.skipif("sys.version_info < (2, 7, 4)")
         def test_os_setgid_error(self):
             os = self.posix
             raises(OverflowError, os.setgid, -2)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -250,6 +250,7 @@
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
     HAVE_UTIMES = rffi_platform.Has('utimes')
+    HAVE_D_TYPE = rffi_platform.Has('DT_UNKNOWN')
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
@@ -603,11 +604,17 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))])
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
+        if HAVE_D_TYPE:
+            DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
+            DT_REG     = rffi_platform.ConstantInteger('DT_REG')
+            DT_DIR     = rffi_platform.ConstantInteger('DT_DIR')
+            DT_LNK     = rffi_platform.ConstantInteger('DT_LNK')
 
     DIRP = rffi.COpaquePtr('DIR')
-    config = rffi_platform.configure(CConfig)
-    DIRENT = config['DIRENT']
+    dirent_config = rffi_platform.configure(CConfig)
+    DIRENT = dirent_config['DIRENT']
     DIRENTP = lltype.Ptr(DIRENT)
     c_opendir = external('opendir',
         [rffi.CCHARP], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
@@ -617,7 +624,9 @@
     # dirent struct (which depends on defines)
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
-    c_closedir = external('closedir', [DIRP], rffi.INT)
+    c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
+else:
+    dirent_config = {}
 
 def _listdir(dirp):
     result = []
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rposix_scandir.py
@@ -0,0 +1,59 @@
+from rpython.rlib import rposix
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.lltypesystem import rffi
+
+
+ at specialize.argtype(0)
+def opendir(path):
+    path = rposix._as_bytes0(path)
+    return opendir_bytes(path)
+
+def opendir_bytes(path):
+    dirp = rposix.c_opendir(path)
+    if not dirp:
+        raise OSError(rposix.get_saved_errno(), "opendir failed")
+    return dirp
+
+def closedir(dirp):
+    rposix.c_closedir(dirp)
+
+def nextentry(dirp):
+    """Read the next entry and returns an opaque object.
+    Use the methods has_xxx() and get_xxx() to read from that
+    opaque object.  The opaque object is valid until the next
+    time nextentry() or closedir() is called.  This may raise
+    StopIteration, or OSError.  Note that this doesn't filter
+    out the "." and ".." entries.
+    """
+    direntp = rposix.c_readdir(dirp)
+    if direntp:
+        return direntp
+    error = rposix.get_saved_errno()
+    if error:
+        raise OSError(error, "readdir failed")
+    raise StopIteration
+
+def has_name_bytes(direntp):
+    return True
+
+def get_name_bytes(direntp):
+    namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
+    return rffi.charp2str(namep)
+
+DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', None)
+DT_REG = rposix.dirent_config.get('DT_REG', None)
+DT_DIR = rposix.dirent_config.get('DT_DIR', None)
+DT_LNK = rposix.dirent_config.get('DT_LNK', None)
+
+def has_type(direntp):
+    return (DT_UNKNOWN is not None and
+            rffi.getintfield(direntp, 'c_d_type') != DT_UNKNOWN)
+
+def type_is_regular(direntp):
+    return rffi.getintfield(direntp, 'c_d_type') == DT_REG
+
+def type_is_dir(direntp):
+    return rffi.getintfield(direntp, 'c_d_type') == DT_DIR
+
+def type_is_link(direntp):
+    return rffi.getintfield(direntp, 'c_d_type') == DT_LNK
diff --git a/rpython/rlib/test/test_rposix_scandir.py b/rpython/rlib/test/test_rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rposix_scandir.py
@@ -0,0 +1,22 @@
+import sys, os
+import py
+from rpython.rlib import rposix_scandir
+
+
+class TestScanDir(object):
+
+    @py.test.mark.skipif("sys.platform == 'win32'")   # XXX
+    def test_name_bytes(self):
+        scan = rposix_scandir.opendir('/')
+        found = []
+        while True:
+            try:
+                p = rposix_scandir.nextentry(scan)
+            except StopIteration:
+                break
+            assert rposix_scandir.has_name_bytes(p)
+            found.append(rposix_scandir.get_name_bytes(p))
+        rposix_scandir.closedir(scan)
+        found.remove('.')
+        found.remove('..')
+        assert sorted(found) == sorted(os.listdir('/'))

From pypy.commits at gmail.com  Sun Aug 21 14:23:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 11:23:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Start on posix.scandir()
Message-ID: <57b9f19b.44ce1c0a.325f4.1dda@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86377:cbc8e2e4d409
Date: 2016-08-21 19:57 +0200
http://bitbucket.org/pypy/pypy/changeset/cbc8e2e4d409/

Log:	Start on posix.scandir()

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -74,6 +74,8 @@
         'abort': 'interp_posix.abort',
         'urandom': 'interp_posix.urandom',
         'device_encoding': 'interp_posix.device_encoding',
+
+        'scandir': 'interp_scandir.scandir',
     }
 
     if hasattr(os, 'chown'):
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -862,13 +862,12 @@
     try:
         path = space.fsencode_w(w_path)
     except OperationError as operr:
+        if operr.async(space):
+            raise
         if not rposix.HAVE_FDOPENDIR:
             raise oefmt(space.w_TypeError,
                 "listdir: illegal type for path argument")
-        if not space.isinstance_w(w_path, space.w_int):
-            raise oefmt(space.w_TypeError,
-                "argument should be string, bytes or integer, not %T", w_path)
-        fd = unwrap_fd(space, w_path)
+        fd = unwrap_fd(space, w_path, "string, bytes or integer")
         try:
             result = rposix.fdlistdir(fd)
         except OSError as e:
diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/posix/interp_scandir.py
@@ -0,0 +1,126 @@
+from rpython.rlib import rgc
+from rpython.rlib import rposix_scandir
+from rpython.rtyper.lltypesystem import lltype
+
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, interp2app
+from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
+from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.baseobjspace import W_Root
+
+from pypy.module.posix.interp_posix import unwrap_fd
+
+
+ at unwrap_spec(w_path=WrappedDefault(u"."))
+def scandir(space, w_path):
+    "scandir(path='.') -> iterator of DirEntry objects for given path"
+
+    if space.isinstance_w(w_path, space.w_bytes):
+        XXX
+    else:
+        try:
+            path_bytes = space.fsencode_w(w_path)
+        except OperationError as operr:
+            if operr.async(space):
+                raise
+            fd = unwrap_fd(space, w_path, "string, bytes or integer")
+            XXXX
+
+        try:
+            dirp = rposix_scandir.opendir(path_bytes)
+        except OSError as e:
+            raise wrap_oserror(space, e)
+        path_prefix = path_bytes
+        if len(path_prefix) > 0 and path_prefix[-1] != '/':
+            path_prefix += '/'
+        w_path_prefix = space.fsdecode(space.newbytes(path_prefix))
+        return W_ScandirIterator(space, dirp, w_path_prefix)
+
+
+class W_ScandirIterator(W_Root):
+    def __init__(self, space, dirp, w_path_prefix):
+        self.space = space
+        self.dirp = dirp
+        self.w_path_prefix = w_path_prefix
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        if self.dirp:
+            rposix_scandir.closedir(self.dirp)
+
+    def iter_w(self):
+        return self.space.wrap(self)
+
+    def fail(self, err=None):
+        dirp = self.dirp
+        if dirp:
+            self.dirp = lltype.nullptr(lltype.typeOf(dirp).TO)
+            rposix_scandir.closedir(dirp)
+        if err is None:
+            raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        else:
+            raise err
+
+    def next_w(self):
+        # XXX not safe against being called on several threads for
+        # the same object, but I think that CPython has the same problem
+        if not self.dirp:
+            self.fail()
+        #
+        space = self.space
+        while True:
+            try:
+                entry = rposix_scandir.nextentry(self.dirp)
+            except StopIteration:
+                self.fail()
+            except OSError as e:
+                self.fail(wrap_oserror(space, e))
+            assert rposix_scandir.has_name_bytes(entry)
+            name = rposix_scandir.get_name_bytes(entry)
+            if name != '.' and name != '..':
+                break
+        #
+        direntry = W_DirEntry(name, self.w_path_prefix)
+        return space.wrap(direntry)
+
+
+W_ScandirIterator.typedef = TypeDef(
+    'posix.ScandirIterator',
+    __iter__ = interp2app(W_ScandirIterator.iter_w),
+    __next__ = interp2app(W_ScandirIterator.next_w),
+)
+W_ScandirIterator.typedef.acceptable_as_base_class = False
+
+
+class W_DirEntry(W_Root):
+    w_name = None
+    w_path = None
+
+    def __init__(self, name_bytes, w_path_prefix):
+        self.name_bytes = name_bytes
+        self.w_path_prefix = w_path_prefix
+
+    def fget_name(self, space):
+        w_name = self.w_name
+        if w_name is None:
+            w_name = space.fsdecode(space.newbytes(self.name_bytes))
+            self.w_name = w_name
+        return w_name
+
+    def fget_path(self, space):
+        w_path = self.w_path
+        if w_path is None:
+            w_name = self.fget_name(space)
+            w_path = space.add(self.w_path_prefix, w_name)
+            self.w_path = w_path
+        return w_path
+
+W_DirEntry.typedef = TypeDef(
+    'posix.DirEntry',
+    name = GetSetProperty(W_DirEntry.fget_name,
+                          doc="the entry's base filename, relative to "
+                              'scandir() "path" argument'),
+    path = GetSetProperty(W_DirEntry.fget_path,
+                          doc="the entry's full path name; equivalent to "
+                              "os.path.join(scandir_path, entry.name)"),
+)
+W_DirEntry.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/posix/test/test_scandir.py
@@ -0,0 +1,49 @@
+import sys, os
+from rpython.tool.udir import udir
+from pypy.module.posix.test import test_posix2
+
+
+def _make_dir(dirname, content):
+    d = os.path.join(str(udir), dirname)
+    os.mkdir(d)
+    for key, value in content.items():
+        xxx
+    return d.decode(sys.getfilesystemencoding())
+
+
+class AppTestScandir(object):
+    spaceconfig = {'usemodules': test_posix2.USEMODULES}
+
+    def setup_class(cls):
+        space = cls.space
+        cls.w_posix = space.appexec([], test_posix2.GET_POSIX)
+        cls.w_dir_empty = space.wrap(_make_dir('empty', {}))
+
+    def test_scandir_empty(self):
+        posix = self.posix
+        sd = posix.scandir(self.dir_empty)
+        assert list(sd) == []
+        assert list(sd) == []
+
+    def test_unicode_versus_bytes(self):
+        posix = self.posix
+        d = next(posix.scandir())
+        assert type(d.name) is str
+        assert type(d.path) is str
+        assert d.path == './' + d.name
+        d = next(posix.scandir(u'.'))
+        assert type(d.name) is str
+        assert type(d.path) is str
+        assert d.path == './' + d.name
+        d = next(posix.scandir(b'.'))
+        assert type(d.name) is bytes
+        assert type(d.path) is bytes
+        assert d.path == b'./' + d.name
+        d = next(posix.scandir('/'))
+        assert type(d.name) is str
+        assert type(d.path) is str
+        assert d.path == '/' + d.name
+        d = next(posix.scandir(b'/'))
+        assert type(d.name) is bytes
+        assert type(d.path) is bytes
+        assert d.path == b'/' + d.name

From pypy.commits at gmail.com  Sun Aug 21 15:13:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 12:13:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: Update rposix_scandir
Message-ID: <57b9fd46.82cbc20a.4def.11ac@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86380:9ef4f79a0ca2
Date: 2016-08-21 21:07 +0200
http://bitbucket.org/pypy/pypy/changeset/9ef4f79a0ca2/

Log:	Update rposix_scandir

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -590,6 +590,8 @@
                                                     "decode", w_fs_encoding)
                 except OperationError as e:
                     # fall back to the original byte string
+                    if e.async(space):
+                        raise
                     result_w[i] = w_bytes
             return space.newlist(result_w)
         else:
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -1,6 +1,6 @@
 from rpython.rlib import rposix
 from rpython.rlib.objectmodel import specialize
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 
 @specialize.argtype(0)
@@ -17,21 +17,22 @@
 def closedir(dirp):
     rposix.c_closedir(dirp)
 
+NULL_DIRP = lltype.nullptr(rposix.DIRENT)
+
 def nextentry(dirp):
     """Read the next entry and returns an opaque object.
     Use the methods has_xxx() and get_xxx() to read from that
     opaque object.  The opaque object is valid until the next
     time nextentry() or closedir() is called.  This may raise
-    StopIteration, or OSError.  Note that this doesn't filter
-    out the "." and ".." entries.
+    OSError, or return a NULL pointer when exhausted.  Note
+    that this doesn't filter out the "." and ".." entries.
     """
     direntp = rposix.c_readdir(dirp)
     if direntp:
-        return direntp
-    error = rposix.get_saved_errno()
-    if error:
-        raise OSError(error, "readdir failed")
-    raise StopIteration
+        error = rposix.get_saved_errno()
+        if error:
+            raise OSError(error, "readdir failed")
+    return direntp
 
 def has_name_bytes(direntp):
     return True
@@ -40,20 +41,12 @@
     namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
     return rffi.charp2str(namep)
 
-DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', None)
-DT_REG = rposix.dirent_config.get('DT_REG', None)
-DT_DIR = rposix.dirent_config.get('DT_DIR', None)
-DT_LNK = rposix.dirent_config.get('DT_LNK', None)
+DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0)
+DT_REG = rposix.dirent_config.get('DT_REG', -1)
+DT_DIR = rposix.dirent_config.get('DT_DIR', -1)
+DT_LNK = rposix.dirent_config.get('DT_LNK', -1)
 
-def has_type(direntp):
-    return (DT_UNKNOWN is not None and
-            rffi.getintfield(direntp, 'c_d_type') != DT_UNKNOWN)
-
-def type_is_regular(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_REG
-
-def type_is_dir(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_DIR
-
-def type_is_link(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_LNK
+def get_known_type(direntp):
+    if rposix.HAVE_D_TYPE:
+        return rffi.getintfield(direntp, 'c_d_type')
+    return DT_UNKNOWN
diff --git a/rpython/rlib/test/test_rposix_scandir.py b/rpython/rlib/test/test_rposix_scandir.py
--- a/rpython/rlib/test/test_rposix_scandir.py
+++ b/rpython/rlib/test/test_rposix_scandir.py
@@ -10,9 +10,8 @@
         scan = rposix_scandir.opendir('/')
         found = []
         while True:
-            try:
-                p = rposix_scandir.nextentry(scan)
-            except StopIteration:
+            p = rposix_scandir.nextentry(scan)
+            if not p:
                 break
             assert rposix_scandir.has_name_bytes(p)
             found.append(rposix_scandir.get_name_bytes(p))

From pypy.commits at gmail.com  Sun Aug 21 15:13:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 12:13:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: Continue working on os.scandir()
 on a branch
Message-ID: <57b9fd47.c2a5c20a.44b61.0fc0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86381:7bb71f9d023d
Date: 2016-08-21 21:07 +0200
http://bitbucket.org/pypy/pypy/changeset/7bb71f9d023d/

Log:	Continue working on os.scandir() on a branch


From pypy.commits at gmail.com  Sun Aug 21 15:13:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 12:13:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57b9fd49.82cbc20a.4def.11b1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86382:6f96b85e2f64
Date: 2016-08-21 21:11 +0200
http://bitbucket.org/pypy/pypy/changeset/6f96b85e2f64/

Log:	hg merge default

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -862,6 +862,8 @@
     try:
         path = space.fsencode_w(w_path)
     except OperationError as operr:
+        if operr.async(space):
+            raise
         if not rposix.HAVE_FDOPENDIR:
             raise oefmt(space.w_TypeError,
                 "listdir: illegal type for path argument")
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -1,6 +1,6 @@
 from rpython.rlib import rposix
 from rpython.rlib.objectmodel import specialize
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 
 @specialize.argtype(0)
@@ -17,21 +17,22 @@
 def closedir(dirp):
     rposix.c_closedir(dirp)
 
+NULL_DIRP = lltype.nullptr(rposix.DIRENT)
+
 def nextentry(dirp):
     """Read the next entry and returns an opaque object.
     Use the methods has_xxx() and get_xxx() to read from that
     opaque object.  The opaque object is valid until the next
     time nextentry() or closedir() is called.  This may raise
-    StopIteration, or OSError.  Note that this doesn't filter
-    out the "." and ".." entries.
+    OSError, or return a NULL pointer when exhausted.  Note
+    that this doesn't filter out the "." and ".." entries.
     """
     direntp = rposix.c_readdir(dirp)
     if direntp:
-        return direntp
-    error = rposix.get_saved_errno()
-    if error:
-        raise OSError(error, "readdir failed")
-    raise StopIteration
+        error = rposix.get_saved_errno()
+        if error:
+            raise OSError(error, "readdir failed")
+    return direntp
 
 def has_name_bytes(direntp):
     return True
@@ -40,20 +41,12 @@
     namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
     return rffi.charp2str(namep)
 
-DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', None)
-DT_REG = rposix.dirent_config.get('DT_REG', None)
-DT_DIR = rposix.dirent_config.get('DT_DIR', None)
-DT_LNK = rposix.dirent_config.get('DT_LNK', None)
+DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0)
+DT_REG = rposix.dirent_config.get('DT_REG', -1)
+DT_DIR = rposix.dirent_config.get('DT_DIR', -1)
+DT_LNK = rposix.dirent_config.get('DT_LNK', -1)
 
-def has_type(direntp):
-    return (DT_UNKNOWN is not None and
-            rffi.getintfield(direntp, 'c_d_type') != DT_UNKNOWN)
-
-def type_is_regular(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_REG
-
-def type_is_dir(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_DIR
-
-def type_is_link(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_LNK
+def get_known_type(direntp):
+    if rposix.HAVE_D_TYPE:
+        return rffi.getintfield(direntp, 'c_d_type')
+    return DT_UNKNOWN
diff --git a/rpython/rlib/test/test_rposix_scandir.py b/rpython/rlib/test/test_rposix_scandir.py
--- a/rpython/rlib/test/test_rposix_scandir.py
+++ b/rpython/rlib/test/test_rposix_scandir.py
@@ -10,9 +10,8 @@
         scan = rposix_scandir.opendir('/')
         found = []
         while True:
-            try:
-                p = rposix_scandir.nextentry(scan)
-            except StopIteration:
+            p = rposix_scandir.nextentry(scan)
+            if not p:
                 break
             assert rposix_scandir.has_name_bytes(p)
             found.append(rposix_scandir.get_name_bytes(p))

From pypy.commits at gmail.com  Sun Aug 21 15:13:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 12:13:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: hg merge py3k
Message-ID: <57b9fd4b.02c41c0a.78b42.71b6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86383:995468378b4c
Date: 2016-08-21 21:11 +0200
http://bitbucket.org/pypy/pypy/changeset/995468378b4c/

Log:	hg merge py3k

diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -1,6 +1,6 @@
 from rpython.rlib import rposix
 from rpython.rlib.objectmodel import specialize
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 
 @specialize.argtype(0)
@@ -17,21 +17,22 @@
 def closedir(dirp):
     rposix.c_closedir(dirp)
 
+NULL_DIRP = lltype.nullptr(rposix.DIRENT)
+
 def nextentry(dirp):
     """Read the next entry and returns an opaque object.
     Use the methods has_xxx() and get_xxx() to read from that
     opaque object.  The opaque object is valid until the next
     time nextentry() or closedir() is called.  This may raise
-    StopIteration, or OSError.  Note that this doesn't filter
-    out the "." and ".." entries.
+    OSError, or return a NULL pointer when exhausted.  Note
+    that this doesn't filter out the "." and ".." entries.
     """
     direntp = rposix.c_readdir(dirp)
     if direntp:
-        return direntp
-    error = rposix.get_saved_errno()
-    if error:
-        raise OSError(error, "readdir failed")
-    raise StopIteration
+        error = rposix.get_saved_errno()
+        if error:
+            raise OSError(error, "readdir failed")
+    return direntp
 
 def has_name_bytes(direntp):
     return True
@@ -40,20 +41,12 @@
     namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
     return rffi.charp2str(namep)
 
-DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', None)
-DT_REG = rposix.dirent_config.get('DT_REG', None)
-DT_DIR = rposix.dirent_config.get('DT_DIR', None)
-DT_LNK = rposix.dirent_config.get('DT_LNK', None)
+DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0)
+DT_REG = rposix.dirent_config.get('DT_REG', -1)
+DT_DIR = rposix.dirent_config.get('DT_DIR', -1)
+DT_LNK = rposix.dirent_config.get('DT_LNK', -1)
 
-def has_type(direntp):
-    return (DT_UNKNOWN is not None and
-            rffi.getintfield(direntp, 'c_d_type') != DT_UNKNOWN)
-
-def type_is_regular(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_REG
-
-def type_is_dir(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_DIR
-
-def type_is_link(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_LNK
+def get_known_type(direntp):
+    if rposix.HAVE_D_TYPE:
+        return rffi.getintfield(direntp, 'c_d_type')
+    return DT_UNKNOWN
diff --git a/rpython/rlib/test/test_rposix_scandir.py b/rpython/rlib/test/test_rposix_scandir.py
--- a/rpython/rlib/test/test_rposix_scandir.py
+++ b/rpython/rlib/test/test_rposix_scandir.py
@@ -10,9 +10,8 @@
         scan = rposix_scandir.opendir('/')
         found = []
         while True:
-            try:
-                p = rposix_scandir.nextentry(scan)
-            except StopIteration:
+            p = rposix_scandir.nextentry(scan)
+            if not p:
                 break
             assert rposix_scandir.has_name_bytes(p)
             found.append(rposix_scandir.get_name_bytes(p))

From pypy.commits at gmail.com  Sun Aug 21 15:13:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 12:13:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: in-progress: is_file() etc.
Message-ID: <57b9fd4d.a6a5c20a.d74e.1a6b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86384:7a230ed585cb
Date: 2016-08-21 21:12 +0200
http://bitbucket.org/pypy/pypy/changeset/7a230ed585cb/

Log:	in-progress: is_file() etc.

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -1,6 +1,5 @@
 from rpython.rlib import rgc
 from rpython.rlib import rposix_scandir
-from rpython.rtyper.lltypesystem import lltype
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, interp2app
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror2
@@ -58,7 +57,7 @@
     def fail(self, err=None):
         dirp = self.dirp
         if dirp:
-            self.dirp = lltype.nullptr(lltype.typeOf(dirp).TO)
+            self.dirp = rposix_scandir.NULL_DIRP
             rposix_scandir.closedir(dirp)
         if err is None:
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
@@ -66,8 +65,8 @@
             raise err
 
     def next_w(self):
-        # XXX not safe against being called on several threads for
-        # the same object, but I think that CPython has the same problem
+        # XXX not safe against being called on several threads for the
+        # same ScandirIterator, but I think that CPython has the same problem
         if not self.dirp:
             self.fail()
         #
@@ -75,20 +74,20 @@
         while True:
             try:
                 entry = rposix_scandir.nextentry(self.dirp)
-            except StopIteration:
-                self.fail()
             except OSError as e:
                 self.fail(wrap_oserror(space, e))
+            if not entry:
+                self.fail()
             assert rposix_scandir.has_name_bytes(entry)
             name = rposix_scandir.get_name_bytes(entry)
             if name != '.' and name != '..':
                 break
         #
+        known_type = rposix_scandir.get_known_type(entry)
         w_name = space.newbytes(name)
-        result_is_bytes = self.result_is_bytes
-        if not result_is_bytes:
+        if not self.result_is_bytes:
             w_name = space.fsdecode(w_name)
-        direntry = W_DirEntry(w_name, self.w_path_prefix, result_is_bytes)
+        direntry = W_DirEntry(w_name, self.w_path_prefix, known_type)
         return space.wrap(direntry)
 
 
@@ -103,10 +102,10 @@
 class W_DirEntry(W_Root):
     w_path = None
 
-    def __init__(self, w_name, w_path_prefix, result_is_bytes):
+    def __init__(self, w_name, w_path_prefix, known_type):
         self.w_name = w_name
         self.w_path_prefix = w_path_prefix
-        self.result_is_bytes = result_is_bytes
+        self.known_type = known_type
 
     def fget_name(self, space):
         return self.w_name
@@ -118,6 +117,44 @@
             self.w_path = w_path
         return w_path
 
+    def is_dir(self, follow_symlinks):
+        known_type = self.known_type
+        if known_type != rposix_scandir.DT_UNKNOWN:
+            if known_type == rposix_scandir.DT_DIR:
+                return True
+            if known_type != rposix_scandir.DT_LNK or not follow_symlinks:
+                return False
+        xxxx
+
+    def is_file(self, follow_symlinks):
+        known_type = self.known_type
+        if known_type != rposix_scandir.DT_UNKNOWN:
+            if known_type == rposix_scandir.DT_REG:
+                return True
+            if known_type != rposix_scandir.DT_LNK or not follow_symlinks:
+                return False
+        xxxx
+
+    def is_symlink(self):
+        known_type = self.known_type
+        if known_type != rposix_scandir.DT_UNKNOWN:
+            return known_type == rposix_scandir.DT_LNK
+        xxxx
+
+    @unwrap_spec(follow_symlinks=int)
+    def descr_is_dir(self, space, __kwonly__, follow_symlinks=1):
+        """return True if the entry is a directory; cached per entry"""
+        return space.wrap(self.is_dir(follow_symlinks))
+
+    @unwrap_spec(follow_symlinks=int)
+    def descr_is_file(self, space, __kwonly__, follow_symlinks=1):
+        """return True if the entry is a file; cached per entry"""
+        return space.wrap(self.is_file(follow_symlinks))
+
+    def descr_is_symlink(self, space):
+        """return True if the entry is a symbolic link; cached per entry"""
+        return space.wrap(self.is_symlink())
+
 W_DirEntry.typedef = TypeDef(
     'posix.DirEntry',
     name = GetSetProperty(W_DirEntry.fget_name,
@@ -126,5 +163,8 @@
     path = GetSetProperty(W_DirEntry.fget_path,
                           doc="the entry's full path name; equivalent to "
                               "os.path.join(scandir_path, entry.name)"),
+    is_dir = interp2app(W_DirEntry.descr_is_dir),
+    is_file = interp2app(W_DirEntry.descr_is_file),
+    is_symlink = interp2app(W_DirEntry.descr_is_symlink),
 )
 W_DirEntry.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -7,7 +7,22 @@
     d = os.path.join(str(udir), dirname)
     os.mkdir(d)
     for key, value in content.items():
-        xxx
+        filename = os.path.join(d, key)
+        if value == 'dir':
+            os.mkdir(filename)
+        elif value == 'file':
+            with open(filename, 'w') as f:
+                pass
+        elif value == 'symlink-file':
+            os.symlink(str(udir.ensure('some_file')), filename)
+        elif value == 'symlink-dir':
+            os.symlink(str(udir), filename)
+        elif value == 'symlink-broken':
+            os.symlink(filename + '-broken', filename)
+        elif value == 'symlink-error':
+            os.symlink(filename, filename)
+        else:
+            raise NotImplementedError(repr(value))
     return d.decode(sys.getfilesystemencoding())
 
 
@@ -18,6 +33,15 @@
         space = cls.space
         cls.w_posix = space.appexec([], test_posix2.GET_POSIX)
         cls.w_dir_empty = space.wrap(_make_dir('empty', {}))
+        cls.w_dir0 = space.wrap(_make_dir('dir0', {'f1': 'file',
+                                                   'f2': 'file',
+                                                   'f3': 'file'}))
+        cls.w_dir1 = space.wrap(_make_dir('dir1', {'file1': 'file'}))
+        cls.w_dir2 = space.wrap(_make_dir('dir2', {'subdir2': 'dir'}))
+        cls.w_dir3 = space.wrap(_make_dir('dir3', {'sfile3': 'symlink-file'}))
+        cls.w_dir4 = space.wrap(_make_dir('dir4', {'sdir4': 'symlink-dir'}))
+        cls.w_dir5 = space.wrap(_make_dir('dir5', {'sbrok5': 'symlink-broken'}))
+        cls.w_dir6 = space.wrap(_make_dir('dir6', {'serr6': 'symlink-error'}))
 
     def test_scandir_empty(self):
         posix = self.posix
@@ -25,6 +49,12 @@
         assert list(sd) == []
         assert list(sd) == []
 
+    def test_scandir_files(self):
+        posix = self.posix
+        sd = posix.scandir(self.dir0)
+        names = [d.name for d in sd]
+        assert sorted(names) == ['f1', 'f2', 'f3']
+
     def test_unicode_versus_bytes(self):
         posix = self.posix
         d = next(posix.scandir())
@@ -47,3 +77,56 @@
         assert type(d.name) is bytes
         assert type(d.path) is bytes
         assert d.path == b'/' + d.name
+
+    def test_dir1(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert d.name == 'file1'
+        assert     d.is_file()
+        assert not d.is_dir()
+        assert not d.is_symlink()
+        raises(TypeError, d.is_file, True)
+
+    def test_dir2(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir2))
+        assert d.name == 'subdir2'
+        assert not d.is_file()
+        assert     d.is_dir()
+        assert not d.is_symlink()
+
+    def test_dir3(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir3))
+        assert d.name == 'sfile3'
+        assert     d.is_file()
+        assert not d.is_dir()
+        assert     d.is_symlink()
+        assert     d.is_file(follow_symlinks=True)
+        assert not d.is_file(follow_symlinks=False)
+
+    def test_dir4(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir4))
+        assert d.name == 'sdir4'
+        assert not d.is_file()
+        assert     d.is_dir()
+        assert     d.is_symlink()
+        assert     d.is_dir(follow_symlinks=True)
+        assert not d.is_dir(follow_symlinks=False)
+
+    def test_dir5(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir5))
+        assert d.name == 'sbrok5'
+        assert not d.is_file()
+        assert not d.is_dir()
+        assert     d.is_symlink()
+
+    def test_dir6(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir6))
+        assert d.name == 'serr6'
+        raises(OSError, d.is_file)
+        raises(OSError, d.is_dir)
+        assert d.is_symlink()

From pypy.commits at gmail.com  Sun Aug 21 17:12:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 14:12:13 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Stopped making a CPython bug
 report for each and every one of them.
Message-ID: <57ba192d.6211c20a.3ee9e.3ea3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5678:7f9daee054b4
Date: 2016-08-21 23:12 +0200
http://bitbucket.org/pypy/extradoc/changeset/7f9daee054b4/

Log:	Stopped making a CPython bug report for each and every one of them.
	Switched to document the crashers in this file for now.

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
new file mode 100644
--- /dev/null
+++ b/planning/py3.5/cpython-crashers.rst
@@ -0,0 +1,24 @@
+CPython crashers
+================
+
+This document ways to crash CPython 3.5, or get completely unexpected
+and undocumented results, or leak memory, etc.
+
+
+* _PyGen_Finalize() should not fail with an exception
+  http://bugs.python.org/issue27811
+
+* PyFrameObject.f_gen can be left pointing to a dangling generator
+  http://bugs.python.org/issue27812
+
+* os.scandir() returns an iterable object that should not be used
+  from multiple threads.  Doing so can e.g. cause one thread to
+  close the dirp while another thread is still using it.  This is
+  likely to crash.  Similarly, the test for (!iterator->dirp) at
+  the start of ScandirIterator_iternext() is only done once even
+  if the following loop runs two or three times because of "." or
+  ".." entries.
+
+* os.scandir() direntry objects should not have stat() called from two
+  threads concurrently.  It will make two stat objects and leak one of
+  them.

From pypy.commits at gmail.com  Sun Aug 21 19:04:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 16:04:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: Misc
Message-ID: <57ba337c.a710c20a.b630c.55db@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86385:0da7aed60c9e
Date: 2016-08-22 01:02 +0200
http://bitbucket.org/pypy/pypy/changeset/0da7aed60c9e/

Log:	Misc

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -625,6 +625,7 @@
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
+    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
 else:
     dirent_config = {}
 
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -42,9 +42,9 @@
     return rffi.charp2str(namep)
 
 DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0)
-DT_REG = rposix.dirent_config.get('DT_REG', -1)
-DT_DIR = rposix.dirent_config.get('DT_DIR', -1)
-DT_LNK = rposix.dirent_config.get('DT_LNK', -1)
+DT_REG = rposix.dirent_config.get('DT_REG', 255)
+DT_DIR = rposix.dirent_config.get('DT_DIR', 255)
+DT_LNK = rposix.dirent_config.get('DT_LNK', 255)
 
 def get_known_type(direntp):
     if rposix.HAVE_D_TYPE:

From pypy.commits at gmail.com  Sun Aug 21 19:04:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 16:04:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: hg merge default
Message-ID: <57ba337e.643ac20a.6057d.5252@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86386:c1499faf06b6
Date: 2016-08-22 01:02 +0200
http://bitbucket.org/pypy/pypy/changeset/c1499faf06b6/

Log:	hg merge default

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -625,6 +625,7 @@
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
+    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
 else:
     dirent_config = {}
 
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -42,9 +42,9 @@
     return rffi.charp2str(namep)
 
 DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0)
-DT_REG = rposix.dirent_config.get('DT_REG', -1)
-DT_DIR = rposix.dirent_config.get('DT_DIR', -1)
-DT_LNK = rposix.dirent_config.get('DT_LNK', -1)
+DT_REG = rposix.dirent_config.get('DT_REG', 255)
+DT_DIR = rposix.dirent_config.get('DT_DIR', 255)
+DT_LNK = rposix.dirent_config.get('DT_LNK', 255)
 
 def get_known_type(direntp):
     if rposix.HAVE_D_TYPE:

From pypy.commits at gmail.com  Sun Aug 21 19:04:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 21 Aug 2016 16:04:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: Finish the first draft of
 os.scandir() (missing: Win32)
Message-ID: <57ba3380.8cc51c0a.2128.6e75@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86387:cfe6bf314382
Date: 2016-08-22 01:03 +0200
http://bitbucket.org/pypy/pypy/changeset/cfe6bf314382/

Log:	Finish the first draft of os.scandir() (missing: Win32)

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -1,12 +1,14 @@
+import stat
+from errno import ENOENT
 from rpython.rlib import rgc
-from rpython.rlib import rposix_scandir
+from rpython.rlib import rposix, rposix_scandir, rposix_stat
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, interp2app
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror2
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.baseobjspace import W_Root
 
-from pypy.module.posix.interp_posix import unwrap_fd
+from pypy.module.posix.interp_posix import unwrap_fd, build_stat_result
 
 
 @unwrap_spec(w_path=WrappedDefault(u"."))
@@ -36,13 +38,18 @@
     w_path_prefix = space.newbytes(path_prefix)
     if not result_is_bytes:
         w_path_prefix = space.fsdecode(w_path_prefix)
-    return W_ScandirIterator(space, dirp, w_path_prefix, result_is_bytes)
+    if rposix.HAVE_FSTATAT:
+        dirfd = rposix.c_dirfd(dirp)
+    else:
+        dirfd = -1
+    return W_ScandirIterator(space, dirp, dirfd, w_path_prefix, result_is_bytes)
 
 
 class W_ScandirIterator(W_Root):
-    def __init__(self, space, dirp, w_path_prefix, result_is_bytes):
+    def __init__(self, space, dirp, dirfd, w_path_prefix, result_is_bytes):
         self.space = space
         self.dirp = dirp
+        self.dirfd = dirfd
         self.w_path_prefix = w_path_prefix
         self.result_is_bytes = result_is_bytes
 
@@ -57,6 +64,7 @@
     def fail(self, err=None):
         dirp = self.dirp
         if dirp:
+            self.dirfd = -1
             self.dirp = rposix_scandir.NULL_DIRP
             rposix_scandir.closedir(dirp)
         if err is None:
@@ -84,10 +92,7 @@
                 break
         #
         known_type = rposix_scandir.get_known_type(entry)
-        w_name = space.newbytes(name)
-        if not self.result_is_bytes:
-            w_name = space.fsdecode(w_name)
-        direntry = W_DirEntry(w_name, self.w_path_prefix, known_type)
+        direntry = W_DirEntry(self, name, known_type)
         return space.wrap(direntry)
 
 
@@ -99,13 +104,31 @@
 W_ScandirIterator.typedef.acceptable_as_base_class = False
 
 
+class FileNotFound(Exception):
+    pass
+
+assert 0 <= rposix_scandir.DT_UNKNOWN <= 255
+assert 0 <= rposix_scandir.DT_REG <= 255
+assert 0 <= rposix_scandir.DT_DIR <= 255
+assert 0 <= rposix_scandir.DT_LNK <= 255
+FLAG_STAT  = 256
+FLAG_LSTAT = 512
+
+
 class W_DirEntry(W_Root):
     w_path = None
 
-    def __init__(self, w_name, w_path_prefix, known_type):
+    def __init__(self, scandir_iterator, name, known_type):
+        self.space = scandir_iterator.space
+        self.scandir_iterator = scandir_iterator
+        self.name = name     # always bytes on Posix
+        self.flags = known_type
+        assert known_type == (known_type & 255)
+        #
+        w_name = self.space.newbytes(name)
+        if not scandir_iterator.result_is_bytes:
+            w_name = self.space.fsdecode(w_name)
         self.w_name = w_name
-        self.w_path_prefix = w_path_prefix
-        self.known_type = known_type
 
     def fget_name(self, space):
         return self.w_name
@@ -113,33 +136,126 @@
     def fget_path(self, space):
         w_path = self.w_path
         if w_path is None:
-            w_path = space.add(self.w_path_prefix, self.w_name)
+            w_path_prefix = self.scandir_iterator.w_path_prefix
+            w_path = space.add(w_path_prefix, self.w_name)
             self.w_path = w_path
         return w_path
 
+    # The internal methods, used to implement the public methods at
+    # the end of the class.  Every method only calls methods *before*
+    # it in program order, so there is no cycle.
+
+    def get_lstat(self):
+        """Get the lstat() of the direntry."""
+        if (self.flags & FLAG_LSTAT) == 0:
+            # Unlike CPython, try to use fstatat() if possible
+            dirfd = self.scandir_iterator.dirfd
+            if dirfd != -1:
+                st = rposix_stat.fstatat(self.name, dirfd,
+                                         follow_symlinks=False)
+            else:
+                path = self.space.fsencode_w(self.fget_path(self.space))
+                st = rposix_stat.lstat(path)
+            self.d_lstat = st
+            self.flags |= FLAG_LSTAT
+        return self.d_lstat
+
+    def get_stat(self):
+        """Get the stat() of the direntry.  This is implemented in
+        such a way that it won't do both a stat() and a lstat().
+        """
+        if (self.flags & FLAG_STAT) == 0:
+            # We don't have the 'd_stat'.  If the known_type says the
+            # direntry is not a DT_LNK, then try to get and cache the
+            # 'd_lstat' instead.  Then, or if we already have a
+            # 'd_lstat' from before, *and* if the 'd_lstat' is not a
+            # S_ISLNK, we can reuse it unchanged for 'd_stat'.
+            #
+            # Note how, in the common case where the known_type says
+            # it is a DT_REG or DT_DIR, then we call and cache lstat()
+            # and that's it.  Also note that in a d_type-less OS or on
+            # a filesystem that always answer DT_UNKNOWN, this method
+            # will instead only call at most stat(), but not cache it
+            # as 'd_lstat'.
+            known_type = self.flags & 255
+            if (known_type != rposix_scandir.DT_UNKNOWN and
+                known_type != rposix_scandir.DT_LNK):
+                self.get_lstat()    # fill the 'd_lstat' cache
+                have_lstat = True
+            else:
+                have_lstat = (self.flags & FLAG_LSTAT) != 0
+
+            if have_lstat:
+                # We have the lstat() but not the stat().  They are
+                # the same, unless the 'd_lstat' is a S_IFLNK.
+                must_call_stat = stat.S_ISLNK(self.d_lstat.st_mode)
+            else:
+                must_call_stat = True
+
+            if must_call_stat:
+                # Must call stat().  Try to use fstatat() if possible
+                dirfd = self.scandir_iterator.dirfd
+                if dirfd != -1:
+                    st = rposix_stat.fstatat(self.name, dirfd,
+                                             follow_symlinks=True)
+                else:
+                    path = self.space.fsencode_w(self.fget_path(self.space))
+                    st = rposix_stat.stat(path)
+            else:
+                st = self.d_lstat
+
+            self.d_stat = st
+            self.flags |= FLAG_STAT
+        return self.d_stat
+
+    def get_stat_or_lstat(self, follow_symlinks):
+        if follow_symlinks:
+            return self.get_stat()
+        else:
+            return self.get_lstat()
+
+    def check_mode(self, follow_symlinks):
+        """Get the stat() or lstat() of the direntry, and return the
+        S_IFMT.  If calling stat()/lstat() gives us ENOENT, return -1
+        instead; it is better to give up and answer "no, not this type"
+        to requests, rather than propagate the error.
+        """
+        try:
+            st = self.get_stat_or_lstat(follow_symlinks)
+        except OSError as e:
+            if e.errno == ENOENT:    # not found
+                return -1
+            raise wrap_oserror2(self.space, e, self.fget_path(self.space))
+        return stat.S_IFMT(st.st_mode)
+
     def is_dir(self, follow_symlinks):
-        known_type = self.known_type
+        known_type = self.flags & 255
         if known_type != rposix_scandir.DT_UNKNOWN:
             if known_type == rposix_scandir.DT_DIR:
                 return True
-            if known_type != rposix_scandir.DT_LNK or not follow_symlinks:
+            elif follow_symlinks and known_type == rposix_scandir.DT_LNK:
+                pass    # don't know in this case
+            else:
                 return False
-        xxxx
+        return self.check_mode(follow_symlinks) == stat.S_IFDIR
 
     def is_file(self, follow_symlinks):
-        known_type = self.known_type
+        known_type = self.flags & 255
         if known_type != rposix_scandir.DT_UNKNOWN:
             if known_type == rposix_scandir.DT_REG:
                 return True
-            if known_type != rposix_scandir.DT_LNK or not follow_symlinks:
+            elif follow_symlinks and known_type == rposix_scandir.DT_LNK:
+                pass    # don't know in this case
+            else:
                 return False
-        xxxx
+        return self.check_mode(follow_symlinks) == stat.S_IFREG
 
     def is_symlink(self):
-        known_type = self.known_type
+        """Check if the direntry is a symlink.  May get the lstat()."""
+        known_type = self.flags & 255
         if known_type != rposix_scandir.DT_UNKNOWN:
             return known_type == rposix_scandir.DT_LNK
-        xxxx
+        return self.check_mode(follow_symlinks=False) == stat.S_IFLNK
 
     @unwrap_spec(follow_symlinks=int)
     def descr_is_dir(self, space, __kwonly__, follow_symlinks=1):
@@ -155,6 +271,13 @@
         """return True if the entry is a symbolic link; cached per entry"""
         return space.wrap(self.is_symlink())
 
+    @unwrap_spec(follow_symlinks=int)
+    def descr_stat(self, space, __kwonly__, follow_symlinks=1):
+        """return stat_result object for the entry; cached per entry"""
+        st = self.get_stat_or_lstat(follow_symlinks)
+        return build_stat_result(self.space, st)
+
+
 W_DirEntry.typedef = TypeDef(
     'posix.DirEntry',
     name = GetSetProperty(W_DirEntry.fget_name,
@@ -166,5 +289,6 @@
     is_dir = interp2app(W_DirEntry.descr_is_dir),
     is_file = interp2app(W_DirEntry.descr_is_file),
     is_symlink = interp2app(W_DirEntry.descr_is_symlink),
+    stat = interp2app(W_DirEntry.descr_stat),
 )
 W_DirEntry.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -78,6 +78,21 @@
         assert type(d.path) is bytes
         assert d.path == b'/' + d.name
 
+    def test_stat1(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert d.name == 'file1'
+        assert d.stat().st_mode & 0o170000 == 0o100000    # S_IFREG
+        assert d.stat().st_size == 0
+
+    def test_stat4(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir4))
+        assert d.name == 'sdir4'
+        assert d.stat().st_mode & 0o170000 == 0o040000    # S_IFDIR
+        assert d.stat(follow_symlinks=True).st_mode &0o170000 == 0o040000
+        assert d.stat(follow_symlinks=False).st_mode&0o170000 == 0o120000 #IFLNK
+
     def test_dir1(self):
         posix = self.posix
         d = next(posix.scandir(self.dir1))
@@ -86,6 +101,8 @@
         assert not d.is_dir()
         assert not d.is_symlink()
         raises(TypeError, d.is_file, True)
+        assert     d.is_file(follow_symlinks=False)
+        assert not d.is_dir(follow_symlinks=False)
 
     def test_dir2(self):
         posix = self.posix
@@ -94,6 +111,8 @@
         assert not d.is_file()
         assert     d.is_dir()
         assert not d.is_symlink()
+        assert not d.is_file(follow_symlinks=False)
+        assert     d.is_dir(follow_symlinks=False)
 
     def test_dir3(self):
         posix = self.posix

From pypy.commits at gmail.com  Sun Aug 21 21:09:48 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 21 Aug 2016 18:09:48 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: __buffer__ is a method
 with a flags arg, not an attibute
Message-ID: <57ba50dc.d32d1c0a.3ca21.8c89@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86388:70768dcd906f
Date: 2016-08-22 11:08 +1000
http://bitbucket.org/pypy/pypy/changeset/70768dcd906f/

Log:	__buffer__ is a method with a flags arg, not an attibute

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,8 +167,8 @@
         else:
             return self.value
 
-    def __buffer__(self):
-        return buffer(self._buffer)
+    def __buffer__(self, flags):
+        return buffer(self._buffer, flags)
 
     def _get_b_base(self):
         try:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -31,9 +31,9 @@
 /* tools to print the array */
 char* stringify(MyArray* a, int nmax){
     char* output = (char*) malloc(nmax * 20);
-    int pos = sprintf(&output[0], "[");
+    int k, pos = sprintf(&output[0], "[");
 
-    for (int k=0; k < a->length && k < nmax; k++){
+    for (k=0; k < a->length && k < nmax; k++){
         pos += sprintf(&output[pos], " %d", a->arr[k]);
     }
     if(a->length > nmax)
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -468,7 +468,7 @@
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_buffer = space.getattr(w_buffer, space.wrap('__buffer__'))
+        w_buffer = space.call_method(w_buffer, '__buffer__', 0)
         buf = _getbuffer(space, w_buffer)
 
     ts = buf.getlength()
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3626,7 +3626,8 @@
         assert str(exc.value) == "assignment destination is read-only"
 
         class A(object):
-            __buffer__ = 'abc'
+            def __buffer__(self, flags):
+                return 'abc'
 
         data = A()
         a = np.frombuffer(data, 'c')
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -464,8 +464,8 @@
         raise oefmt(space.w_TypeError,
                     "Cannot use string as modifiable buffer")
 
-    def descr_getbuffer(self, space):
-        return self
+    def descr_getbuffer(self, space, w_flags):
+        return StringBuffer(self._value)
 
     charbuf_w = str_w
 
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -4,7 +4,7 @@
     def test_init(self):
         import sys
         class A(object):
-            def __buffer__(self):
+            def __buffer__(self, flags):
                 return buffer('123')
         if '__pypy__' not in sys.builtin_module_names:
             raises(TypeError, buffer, A())

From pypy.commits at gmail.com  Mon Aug 22 03:25:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 00:25:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: Protect against usage in multiple
 threads
Message-ID: <57baa8d8.d41a1c0a.7323d.ac0f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86389:b0234181af4b
Date: 2016-08-22 09:24 +0200
http://bitbucket.org/pypy/pypy/changeset/b0234181af4b/

Log:	Protect against usage in multiple threads

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -46,6 +46,8 @@
 
 
 class W_ScandirIterator(W_Root):
+    _in_next = False
+
     def __init__(self, space, dirp, dirfd, w_path_prefix, result_is_bytes):
         self.space = space
         self.dirp = dirp
@@ -73,25 +75,30 @@
             raise err
 
     def next_w(self):
-        # XXX not safe against being called on several threads for the
-        # same ScandirIterator, but I think that CPython has the same problem
         if not self.dirp:
             self.fail()
-        #
-        space = self.space
-        while True:
-            try:
-                entry = rposix_scandir.nextentry(self.dirp)
-            except OSError as e:
-                self.fail(wrap_oserror(space, e))
-            if not entry:
-                self.fail()
-            assert rposix_scandir.has_name_bytes(entry)
-            name = rposix_scandir.get_name_bytes(entry)
-            if name != '.' and name != '..':
-                break
-        #
-        known_type = rposix_scandir.get_known_type(entry)
+        if self._in_next:
+            self.fail(oefmt(self.space.w_RuntimeError,
+               "cannot use ScandirIterator from multiple threads concurrently"))
+        self._in_next = True
+        try:
+            #
+            space = self.space
+            while True:
+                try:
+                    entry = rposix_scandir.nextentry(self.dirp)
+                except OSError as e:
+                    self.fail(wrap_oserror(space, e))
+                if not entry:
+                    self.fail()
+                assert rposix_scandir.has_name_bytes(entry)
+                name = rposix_scandir.get_name_bytes(entry)
+                if name != '.' and name != '..':
+                    break
+            #
+            known_type = rposix_scandir.get_known_type(entry)
+        finally:
+            self._in_next = False
         direntry = W_DirEntry(self, name, known_type)
         return space.wrap(direntry)
 

From pypy.commits at gmail.com  Mon Aug 22 04:32:09 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 22 Aug 2016 01:32:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: merge py3.5
Message-ID: <57bab889.05371c0a.afb9f.03b0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86390:cb2cf459a284
Date: 2016-08-22 09:27 +0200
http://bitbucket.org/pypy/pypy/changeset/cb2cf459a284/

Log:	merge py3.5

diff too long, truncating to 2000 out of 3687 lines

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -99,14 +99,16 @@
     RegrTest('test___all__.py', core=True),
     RegrTest('test___future__.py', core=True),
     RegrTest('test__locale.py', usemodules='_locale'),
+    RegrTest('test__opcode.py'),
     RegrTest('test__osx_support.py'),
     RegrTest('test_abc.py'),
     RegrTest('test_abstract_numbers.py'),
     RegrTest('test_aifc.py'),
     RegrTest('test_argparse.py', usemodules='binascii'),
     RegrTest('test_array.py', core=True, usemodules='struct array binascii'),
+    RegrTest('test_asdl_parser.py'),
     RegrTest('test_ast.py', core=True, usemodules='struct'),
-    RegrTest('test_asynchat.py', usemodules='select fcntl'),
+    RegrTest('test_asynchat.py', usemodules='select fcntl', skip="XXX: deadlocks or takes forever"),
     RegrTest('test_asyncore.py', usemodules='select fcntl'),
     RegrTest('test_atexit.py', core=True),
     RegrTest('test_audioop.py'),
@@ -123,7 +125,7 @@
     RegrTest('test_bufio.py', core=True),
     RegrTest('test_builtin.py', core=True, usemodules='binascii'),
     RegrTest('test_bytes.py', usemodules='struct binascii'),
-    RegrTest('test_bz2.py', usemodules='bz2'),
+    RegrTest('test_bz2.py', usemodules='bz2', skip="XXX: deadlocks?"),
     RegrTest('test_calendar.py'),
     RegrTest('test_call.py', core=True),
     RegrTest('test_capi.py', usemodules='cpyext'),
@@ -134,7 +136,7 @@
     RegrTest('test_cmath.py', core=True),
     RegrTest('test_cmd.py'),
     RegrTest('test_cmd_line.py'),
-    RegrTest('test_cmd_line_script.py'),
+    RegrTest('test_cmd_line_script.py', skip="XXX: deadlocks?"),
     RegrTest('test_code.py', core=True),
     RegrTest('test_code_module.py'),
     RegrTest('test_codeccallbacks.py', core=True),
@@ -151,20 +153,19 @@
     RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'),
     RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec struct unicodedata array'),
     RegrTest('test_codeop.py', core=True),
-    RegrTest('test_coding.py', core=True),
     RegrTest('test_collections.py', usemodules='binascii struct'),
     RegrTest('test_colorsys.py'),
     RegrTest('test_compare.py', core=True),
     RegrTest('test_compile.py', core=True),
     RegrTest('test_compileall.py'),
     RegrTest('test_complex.py', core=True),
-    RegrTest('test_concurrent_futures.py',
-             skip="XXX: deadlocks" if sys.platform == 'win32' else False),
+    RegrTest('test_concurrent_futures.py', skip="XXX: deadlocks" if sys.platform == 'win32' else False),
     RegrTest('test_configparser.py'),
     RegrTest('test_contains.py', core=True),
     RegrTest('test_contextlib.py', usemodules="thread"),
     RegrTest('test_copy.py', core=True),
     RegrTest('test_copyreg.py', core=True),
+    RegrTest('test_coroutines.py'),
     RegrTest('test_cprofile.py'),
     RegrTest('test_crashers.py'),
     RegrTest('test_crypt.py', usemodules='crypt'),
@@ -176,7 +177,7 @@
     RegrTest('test_dbm_dumb.py'),
     RegrTest('test_dbm_gnu.py'),
     RegrTest('test_dbm_ndbm.py'),
-    RegrTest('test_decimal.py'),
+    RegrTest('test_decimal.py', skip="XXX: deadlocks?"),
     RegrTest('test_decorators.py', core=True),
     RegrTest('test_defaultdict.py', usemodules='_collections'),
     RegrTest('test_deque.py', core=True, usemodules='_collections struct'),
@@ -195,8 +196,11 @@
     RegrTest('test_dummy_thread.py', core=True),
     RegrTest('test_dummy_threading.py', core=True),
     RegrTest('test_dynamic.py'),
+    RegrTest('test_dynamicclassattribute.py'),
+    RegrTest('test_eintr.py'),
     RegrTest('test_email', skip="XXX is a directory"),
     RegrTest('test_ensurepip.py'),
+    RegrTest('test_enum.py'),
     RegrTest('test_enumerate.py', core=True),
     RegrTest('test_eof.py', core=True),
     RegrTest('test_epoll.py'),
@@ -204,23 +208,24 @@
     RegrTest('test_exception_variations.py'),
     RegrTest('test_exceptions.py', core=True),
     RegrTest('test_extcall.py', core=True),
-    RegrTest('test_faulthandler.py'),
+    RegrTest('test_faulthandler.py', skip="XXX: deadlocks?"),
     RegrTest('test_fcntl.py', usemodules='fcntl'),
     RegrTest('test_file.py', usemodules="posix", core=True),
     RegrTest('test_file_eintr.py'),
     RegrTest('test_filecmp.py', core=True),
     RegrTest('test_fileinput.py', core=True),
     RegrTest('test_fileio.py'),
+    RegrTest('test_finalization.py'),
     RegrTest('test_float.py', core=True),
     RegrTest('test_flufl.py'),
     RegrTest('test_fnmatch.py', core=True),
-    RegrTest('test_fork1.py', usemodules="thread"),
+    RegrTest('test_fork1.py', usemodules="thread", skip="XXX: deadlocks?"),
     RegrTest('test_format.py', core=True),
     RegrTest('test_fractions.py'),
-    RegrTest('test_frozen.py', skip="unsupported extension module"),
-    RegrTest('test_ftplib.py'),
+    RegrTest('test_frame.py'),
+    RegrTest('test_ftplib.py', skip="XXX: deadlocks?"),
     RegrTest('test_funcattrs.py', core=True),
-    RegrTest('test_functools.py'),
+    RegrTest('test_functools.py', skip="XXX: deadlocks?"),
     RegrTest('test_future.py', core=True),
     RegrTest('test_future3.py', core=True),
     RegrTest('test_future4.py', core=True),
@@ -250,11 +255,9 @@
     RegrTest('test_httplib.py'),
     RegrTest('test_httpservers.py'),
     RegrTest('test_idle.py'),
-    RegrTest('test_imaplib.py'),
+    RegrTest('test_imaplib.py', skip="XXX: deadlocks?"),
     RegrTest('test_imghdr.py'),
     RegrTest('test_imp.py', core=True, usemodules='thread'),
-    RegrTest('test_import.py', core=True),
-    RegrTest('test_importhooks.py', core=True),
     RegrTest('test_importlib', 'XXX is a directory'),
     RegrTest('test_index.py'),
     RegrTest('test_inspect.py', usemodules="struct unicodedata"),
@@ -268,6 +271,7 @@
     RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"),
     RegrTest('test_itertools.py', core=True, usemodules="itertools struct"),
     RegrTest('test_json', skip="XXX is a directory"),
+    RegrTest('test_keyword.py'),
     RegrTest('test_keywordonlyarg.py'),
     RegrTest('test_kqueue.py'),
     RegrTest('test_largefile.py'),
@@ -296,8 +300,10 @@
     RegrTest('test_modulefinder.py'),
     RegrTest('test_msilib.py'),
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
-    RegrTest('test_multiprocessing.py', skip="XXX: deadlocks the buildbots"),
-    RegrTest('test_namespace_pkgs.py'),
+    RegrTest('test_multiprocessing_fork.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_forkserver.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_main_handling.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_spawn.py', skip="XXX: deadlocks?"),
     RegrTest('test_netrc.py'),
     RegrTest('test_nis.py'),
     RegrTest('test_nntplib.py'),
@@ -312,10 +318,10 @@
     RegrTest('test_ossaudiodev.py'),
     RegrTest('test_osx_env.py'),
     RegrTest('test_parser.py', skip="slowly deprecating compiler"),
-    RegrTest('test_pdb.py'),
+    RegrTest('test_pathlib.py'),
+    RegrTest('test_pdb.py', skip="XXX: deadlocks?"),
     RegrTest('test_peepholer.py'),
     RegrTest('test_pep247.py'),
-    RegrTest('test_pep263.py'),
     RegrTest('test_pep277.py'),
     RegrTest('test_pep292.py'),
     RegrTest('test_pep3120.py'),
@@ -323,6 +329,7 @@
     RegrTest('test_pep3151.py'),
     RegrTest('test_pep352.py'),
     RegrTest('test_pep380.py'),
+    RegrTest('test_pep479.py'),
     RegrTest('test_pickle.py', core=True),
     RegrTest('test_pickletools.py', core=False),
     RegrTest('test_pipes.py'),
@@ -331,9 +338,9 @@
     RegrTest('test_pkgutil.py'),
     RegrTest('test_platform.py'),
     RegrTest('test_plistlib.py'),
-    RegrTest('test_poll.py'),
+    RegrTest('test_poll.py', skip="XXX: deadlocks?"),
     RegrTest('test_popen.py'),
-    RegrTest('test_poplib.py'),
+    RegrTest('test_poplib.py', skip="XXX: deadlocks?"),
     RegrTest('test_posix.py', usemodules="_rawffi"),
     RegrTest('test_posixpath.py'),
     RegrTest('test_pow.py', core=True),
@@ -356,6 +363,7 @@
     RegrTest('test_range.py', core=True),
     RegrTest('test_re.py', core=True),
     RegrTest('test_readline.py'),
+    RegrTest('test_regrtest.py'),
     RegrTest('test_reprlib.py', core=True),
     RegrTest('test_resource.py'),
     RegrTest('test_richcmp.py', core=True),
@@ -365,7 +373,9 @@
     RegrTest('test_sax.py'),
     RegrTest('test_sched.py'),
     RegrTest('test_scope.py', core=True),
+    RegrTest('test_script_helper.py'),
     RegrTest('test_select.py'),
+    RegrTest('test_selectors.py'),
     RegrTest('test_set.py', core=True),
     RegrTest('test_setcomps.py', core=True),
     RegrTest('test_shelve.py'),
@@ -381,10 +391,13 @@
     RegrTest('test_socket.py', usemodules='thread _weakref'),
     RegrTest('test_socketserver.py', usemodules='thread'),
     RegrTest('test_sort.py', core=True),
-    RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"),
+    RegrTest('test_source_encoding.py'),
+    RegrTest('test_spwd.py'),
+    RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib", skip="XXX: deadlocks?"),
     RegrTest('test_ssl.py', usemodules='_ssl _socket select'),
     RegrTest('test_startfile.py'),
     RegrTest('test_stat.py'),
+    RegrTest('test_statistics.py'),
     RegrTest('test_strftime.py'),
     RegrTest('test_string.py', core=True),
     RegrTest('test_stringprep.py'),
@@ -394,13 +407,13 @@
     RegrTest('test_struct.py', usemodules='struct'),
     RegrTest('test_structmembers.py', skip="CPython specific"),
     RegrTest('test_structseq.py'),
-    RegrTest('test_subprocess.py', usemodules='signal'),
+    RegrTest('test_subprocess.py', usemodules='signal', skip="XXX: deadlocks?"),
     RegrTest('test_sunau.py'),
     RegrTest('test_sundry.py'),
     RegrTest('test_super.py', core=True),
     RegrTest('test_support.py'),
     RegrTest('test_symtable.py', skip="implementation detail"),
-    RegrTest('test_syntax.py', core=True),
+    RegrTest('test_syntax.py', core=True, skip="XXX: infinite loop?"),
     RegrTest('test_sys.py', core=True, usemodules='struct'),
     RegrTest('test_sys_setprofile.py', core=True),
     RegrTest('test_sys_settrace.py', core=True),
@@ -408,29 +421,30 @@
     RegrTest('test_syslog.py'),
     RegrTest('test_tarfile.py'),
     RegrTest('test_tcl.py'),
-    RegrTest('test_telnetlib.py'),
+    RegrTest('test_telnetlib.py', skip="XXX: deadlocks?"),
     RegrTest('test_tempfile.py'),
     RegrTest('test_textwrap.py'),
     RegrTest('test_thread.py', usemodules="thread", core=True),
-    RegrTest('test_threaded_import.py', usemodules="thread", core=True),
-    RegrTest('test_threadedtempfile.py',
-             usemodules="thread", core=False),
-    RegrTest('test_threading.py', usemodules="thread", core=True),
-    RegrTest('test_threading_local.py', usemodules="thread", core=True),
+    RegrTest('test_threaded_import.py', usemodules="thread", core=True, skip="XXX: deadlocks?"),
+    RegrTest('test_threadedtempfile.py', usemodules="thread", core=False, skip="XXX: deadlocks?"),
+    RegrTest('test_threading.py', usemodules="thread", core=True, skip="XXX: deadlocks?"),
+    RegrTest('test_threading_local.py', usemodules="thread", core=True, skip="XXX: deadlocks?"),
     RegrTest('test_threadsignals.py', usemodules="thread"),
     RegrTest('test_time.py', core=True, usemodules="struct thread _rawffi"),
     RegrTest('test_timeit.py'),
     RegrTest('test_timeout.py'),
+    RegrTest('test_tix.py'),
     RegrTest('test_tk.py'),
     RegrTest('test_tokenize.py'),
-    RegrTest('test_tools.py'),
     RegrTest('test_trace.py'),
     RegrTest('test_traceback.py', core=True),
+    RegrTest('test_tracemalloc.py'),
     RegrTest('test_ttk_guionly.py'),
     RegrTest('test_ttk_textonly.py'),
     RegrTest('test_tuple.py', core=True),
     RegrTest('test_typechecks.py'),
     RegrTest('test_types.py', core=True),
+    RegrTest('test_typing.py'),
     RegrTest('test_ucn.py'),
     RegrTest('test_unary.py', core=True),
     RegrTest('test_unicode.py', core=True),
@@ -455,7 +469,6 @@
     RegrTest('test_venv.py', usemodules="struct"),
     RegrTest('test_wait3.py', usemodules="thread"),
     RegrTest('test_wait4.py', usemodules="thread"),
-    RegrTest('test_warnings.py', core=True),
     RegrTest('test_wave.py'),
     RegrTest('test_weakref.py', core=True, usemodules='_weakref'),
     RegrTest('test_weakset.py'),
@@ -470,8 +483,9 @@
     RegrTest('test_xml_etree_c.py'),
     RegrTest('test_xmlrpc.py'),
     RegrTest('test_xmlrpc_net.py'),
+    RegrTest('test_zipapp.py'),
     RegrTest('test_zipfile.py'),
-    RegrTest('test_zipfile64.py'),
+    RegrTest('test_zipfile64.py', skip="XXX: infinite loop or just long time?"),
     RegrTest('test_zipimport.py', usemodules='zlib zipimport'),
     RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'),
     RegrTest('test_zlib.py', usemodules='zlib'),
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -356,6 +356,11 @@
 that a C-level traceback is usually of no help at all in PyPy.
 Debugging PyPy can be annoying.
 
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
 In more details:
 
 * First, please give the exact PyPy version, and the OS.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -128,3 +128,19 @@
 .. branch: cpyext-realloc
 
 Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -389,7 +389,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -397,9 +398,17 @@
         for block in blocks:
             depth = self._do_stack_depth_walk(block)
             if block.auto_inserted_return and depth != 0:
-                os.write(2, "StackDepthComputationError in %s at %s:%s\n" % (
-                    self.compile_info.filename, self.name, self.first_lineno))
-                raise StackDepthComputationError   # fatal error
+                # This case occurs if this code object uses some
+                # construction for which the stack depth computation
+                # is wrong (too high).  If you get here while working
+                # on the astcompiler, then you should at first ignore
+                # the error, and comment out the 'raise' below.  Such
+                # an error is not really bad: it is just a bit
+                # wasteful.  For release-ready versions, though, we'd
+                # like not to be wasteful. :-)
+                os.write(2, "StackDepthComputationError(POS) in %s at %s:%s\n"
+                  % (self.compile_info.filename, self.name, self.first_lineno))
+                raise StackDepthComputationError   # would-be-nice-not-to-have
         return self._max_depth
 
     def _next_stack_depth_walk(self, nextblock, depth):
@@ -408,8 +417,20 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            if depth < 0:
+                # This is really a fatal error, don't comment out this
+                # 'raise'.  It means that the stack depth computation
+                # thinks there is a path that yields a negative stack
+                # depth, which means that it underestimates the space
+                # needed and it would crash when interpreting this
+                # code.
+                os.write(2, "StackDepthComputationError(NEG) in %s at %s:%s\n"
+                  % (self.compile_info.filename, self.name, self.first_lineno))
+                raise StackDepthComputationError   # really fatal error
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode
@@ -560,7 +581,6 @@
     ops.LIST_APPEND: -1,
     ops.SET_ADD: -1,
     ops.MAP_ADD: -2,
-    # XXX 
 
     ops.BINARY_POWER: -1,
     ops.BINARY_MULTIPLY: -1,
@@ -602,8 +622,8 @@
 
     ops.PRINT_EXPR: -1,
 
-    ops.WITH_CLEANUP_START: -1,
-    ops.WITH_CLEANUP_FINISH: -1,  # XXX Sometimes more
+    ops.WITH_CLEANUP_START: 1,
+    ops.WITH_CLEANUP_FINISH: -2,
     ops.LOAD_BUILD_CLASS: 1,
     ops.POP_BLOCK: 0,
     ops.POP_EXCEPT: -1,
@@ -619,7 +639,6 @@
     ops.YIELD_FROM: -1,
     ops.COMPARE_OP: -1,
 
-    # TODO 
     ops.LOOKUP_METHOD: 1,
 
     ops.LOAD_NAME: 1,
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -599,17 +599,21 @@
         b_try_cleanup = self.new_block()
         b_after_loop = self.new_block()
         b_after_loop_else = self.new_block()
+        
         self.emit_jump(ops.SETUP_LOOP, b_after_loop)
         self.push_frame_block(F_BLOCK_LOOP, b_try)
+        
         fr.iter.walkabout(self)
         self.emit_op(ops.GET_AITER)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
+        
         self.use_next_block(b_try)
         # This adds another line, so each for iteration can be traced.
         self.lineno_set = False
         self.emit_jump(ops.SETUP_EXCEPT, b_except)
         self.push_frame_block(F_BLOCK_EXCEPT, b_try)
+        
         self.emit_op(ops.GET_ANEXT)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
@@ -617,9 +621,10 @@
         self.emit_op(ops.POP_BLOCK)
         self.pop_frame_block(F_BLOCK_EXCEPT, b_try)
         self.emit_jump(ops.JUMP_FORWARD, b_after_try)
+        
         self.use_next_block(b_except)
-        self.emit_op(ops.POP_TOP)
-        self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopIterError")
+        self.emit_op(ops.DUP_TOP)
+        self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopAsyncIteration")
         self.emit_op_arg(ops.COMPARE_OP, 10)
         self.emit_jump(ops.POP_JUMP_IF_FALSE, b_try_cleanup, True)
         
@@ -627,18 +632,29 @@
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_EXCEPT) # for SETUP_EXCEPT
+        # Manually remove the 'aiter' object from the valuestack.
+        # This POP_TOP is not needed from the point of view of
+        # pyopcode.py, which will pop anything to match the stack
+        # depth of the SETUP_LOOP, but it is needed to make
+        # PythonCodeMaker._stacksize() compute an exact result and not
+        # crash with StackDepthComputationError.
+        self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP
         self.emit_jump(ops.JUMP_ABSOLUTE, b_after_loop_else, True)
         
         self.use_next_block(b_try_cleanup)
         self.emit_op(ops.END_FINALLY)
+        
         self.use_next_block(b_after_try)
         self.visit_sequence(fr.body)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_try, True)
+        
         self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP
         self.pop_frame_block(F_BLOCK_LOOP, b_try)
+        
         self.use_next_block(b_after_loop)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_end, True)
+        
         self.use_next_block(b_after_loop_else)
         self.visit_sequence(fr.orelse)
         
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -118,7 +118,7 @@
     # don't constant-fold if "w_left" and "w_right" are integers and
     # the estimated bit length of the power is unreasonably large
     space.appexec([w_left, w_right], """(left, right):
-        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+        if isinstance(left, int) and isinstance(right, int):
             if left.bit_length() * right > 5000:
                 raise OverflowError
     """)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -511,6 +511,9 @@
         x *= 7
         """, 'x', 42
 
+    def test_with_stacksize_bug(self):
+        compile_with_astcompiler("with a:\n  pass", 'exec', self.space)
+
     def test_with_bug(self):
         yield self.simple_test, """
         class ContextManager:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1749,6 +1749,23 @@
                 "Python int too large for C unsigned short")
         return value
 
+    def c_uid_t_w(self, w_obj):
+        # xxx assumes that uid_t and gid_t are a C unsigned int.
+        # Equivalent to space.c_uint_w(), with the exception that
+        # it also accepts -1 and converts that to UINT_MAX, which
+        # is (uid_t)-1.  And values smaller than -1 raise
+        # OverflowError, not ValueError.
+        try:
+            return self.c_uint_w(w_obj)
+        except OperationError as e:
+            if e.match(self, self.w_ValueError):
+                # ValueError: cannot convert negative integer to unsigned
+                if self.int_w(w_obj) == -1:
+                    return UINT_MAX
+                raise oefmt(self.w_OverflowError,
+                            "user/group id smaller than minimum (-1)")
+            raise
+
     def truncatedint_w(self, w_obj, allow_conversion=True):
         # Like space.gateway_int_w(), but return the integer truncated
         # instead of raising OverflowError.  For obscure cases only.
@@ -1937,6 +1954,7 @@
     'PendingDeprecationWarning',
     'ReferenceError',
     'ResourceWarning',
+    'RecursionError',
     'RuntimeError',
     'RuntimeWarning',
     'StopIteration',
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -40,9 +40,6 @@
         and possibly more locals."""
         return self.signature().getallvarnames()
 
-    def getformalargcount(self):
-        return self.signature().scope_length()
-
     def getdocstring(self, space):
         return space.w_None
 
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -38,7 +38,9 @@
                           'name?',
                           'w_kw_defs?']
 
-    def __init__(self, space, code, w_globals=None, defs_w=[], w_kw_defs=None,
+    w_kw_defs = None
+
+    def __init__(self, space, code, w_globals=None, defs_w=[], kw_defs_w=None,
                  closure=None, w_ann=None, forcename=None, qualname=None):
         self.space = space
         self.name = forcename or code.co_name
@@ -48,10 +50,12 @@
         self.w_func_globals = w_globals  # the globals dictionary
         self.closure = closure    # normally, list of Cell instances or None
         self.defs_w = defs_w
-        self.w_kw_defs = w_kw_defs
         self.w_func_dict = None # filled out below if needed
         self.w_module = None
         self.w_ann = w_ann
+        #
+        if kw_defs_w is not None:
+            self.init_kwdefaults_dict(kw_defs_w)
 
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
@@ -379,14 +383,23 @@
 
     def fset_func_kwdefaults(self, space, w_new):
         if space.is_w(w_new, space.w_None):
-            w_new = None
-        elif not space.isinstance_w(w_new, space.w_dict):
-            raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
-        self.w_kw_defs = w_new
+            self.w_kw_defs = None
+        else:
+            if not space.isinstance_w(w_new, space.w_dict):
+                raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
+            self.w_kw_defs = w_new
 
     def fdel_func_kwdefaults(self, space):
         self.w_kw_defs = None
 
+    def init_kwdefaults_dict(self, kw_defs_w):
+        # use the moduledict logic to get normally-constant entries
+        space = self.space
+        w_dict = space.newdict(module=True)
+        for w_name, w_value in kw_defs_w:
+            space.setitem(w_dict, w_name, w_value)
+        self.w_kw_defs = w_dict
+
     def fget_func_doc(self, space):
         if self.w_doc is None:
             self.w_doc = self.code.getdocstring(space)
@@ -663,10 +676,12 @@
     def __init__(self, func):
         assert isinstance(func, Function)
         Function.__init__(self, func.space, func.code, func.w_func_globals,
-                          func.defs_w, None, func.closure, None, func.name)
+                          func.defs_w, None, func.closure,
+                          None, func.name)
         self.w_doc = func.w_doc
         self.w_func_dict = func.w_func_dict
         self.w_module = func.w_module
+        self.w_kw_defs = func.w_kw_defs
 
     def descr_builtinfunction__new__(space, w_subtype):
         raise oefmt(space.w_TypeError,
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -28,6 +28,8 @@
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
 
+NO_DEFAULT = object()
+
 
 # internal non-translatable parts:
 class SignatureBuilder(object):
@@ -44,12 +46,21 @@
         self.argnames = argnames
         self.varargname = varargname
         self.kwargname = kwargname
+        self.kwonlyargnames = None
 
     def append(self, argname):
-        self.argnames.append(argname)
+        if self.kwonlyargnames is None:
+            self.argnames.append(argname)
+        else:
+            self.kwonlyargnames.append(argname)
+
+    def marker_kwonly(self):
+        assert self.kwonlyargnames is None
+        self.kwonlyargnames = []
 
     def signature(self):
-        return Signature(self.argnames, self.varargname, self.kwargname)
+        return Signature(self.argnames, self.varargname, self.kwargname,
+                         self.kwonlyargnames)
 
 #________________________________________________________________
 
@@ -66,13 +77,6 @@
         """NOT_RPYTHON"""
         raise NotImplementedError
 
-def kwonly(arg_unwrapper):
-    """Mark argument as keyword-only.
-
-    XXX: has no actual effect for now.
-    """
-    return arg_unwrapper
-
 
 class UnwrapSpecRecipe(object):
     "NOT_RPYTHON"
@@ -177,6 +181,9 @@
     def visit_c_ushort(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_c_uid_t(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_truncatedint_w(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -226,6 +233,11 @@
         name = int_unwrapping_space_method(typ)
         self.checked_space_method(name, app_sig)
 
+    def visit_kwonly(self, _, app_sig):
+        argname = self.orig_arg()
+        assert argname == '__kwonly__'
+        app_sig.marker_kwonly()
+
 
 class UnwrapSpec_EmitRun(UnwrapSpecEmit):
 
@@ -307,9 +319,15 @@
     def visit_c_ushort(self, typ):
         self.run_args.append("space.c_ushort_w(%s)" % (self.scopenext(),))
 
+    def visit_c_uid_t(self, typ):
+        self.run_args.append("space.c_uid_t_w(%s)" % (self.scopenext(),))
+
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
+    def visit_kwonly(self, typ):
+        self.run_args.append("None")
+
     def _make_unwrap_activation_class(self, unwrap_spec, cache={}):
         try:
             key = tuple(unwrap_spec)
@@ -456,9 +474,15 @@
     def visit_c_ushort(self, typ):
         self.unwrap.append("space.c_ushort_w(%s)" % (self.nextarg(),))
 
+    def visit_c_uid_t(self, typ):
+        self.unwrap.append("space.c_uid_t_w(%s)" % (self.nextarg(),))
+
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
+    def visit_kwonly(self, typ):
+        raise FastFuncNotSupported
+
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
         unwrap_info.apply_over(unwrap_spec)
@@ -554,6 +578,8 @@
                 unwrap_spec.append('args_w')
             elif argname.startswith('w_'):
                 unwrap_spec.append(W_Root)
+            elif argname == '__kwonly__':
+                unwrap_spec.append('kwonly')
             else:
                 unwrap_spec.append(None)
 
@@ -603,7 +629,12 @@
 
         # First extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(func.func_code)
+        sig = pycode.cpython_code_signature(func.func_code)
+        argnames = sig.argnames
+        varargname = sig.varargname
+        kwargname = sig.kwargname
+        if sig.kwonlyargnames:
+            import pdb; pdb.set_trace()
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -627,7 +658,9 @@
         app_sig = SignatureBuilder(func)
 
         UnwrapSpec_Check(orig_sig).apply_over(unwrap_spec, app_sig)
-        self.sig = argnames, varargname, kwargname = app_sig.signature()
+        self.sig = app_sig.signature()
+        argnames = self.sig.argnames
+        varargname = self.sig.varargname
 
         self.minargs = len(argnames)
         if varargname:
@@ -720,7 +753,7 @@
             raise OperationError(space.w_MemoryError, space.w_None)
         except rstackovf.StackOverflow as e:
             rstackovf.check_stack_overflow()
-            raise oefmt(space.w_RuntimeError,
+            raise oefmt(space.w_RecursionError,
                         "maximum recursion depth exceeded")
         except RuntimeError:   # not on top of py.py
             raise OperationError(space.w_RuntimeError, space.w_None)
@@ -936,64 +969,71 @@
         self.name = app_name
         self.as_classmethod = as_classmethod
 
-        if not f.func_defaults:
-            self._staticdefs = []
-        else:
-            argnames = self._code._argnames
-            defaults = f.func_defaults
-            self._staticdefs = zip(argnames[-len(defaults):], defaults)
+        argnames = self._code._argnames
+        defaults = f.func_defaults or ()
+        self._staticdefs = dict(zip(
+            argnames[len(argnames) - len(defaults):], defaults))
+
         return self
 
     def _getdefaults(self, space):
         "NOT_RPYTHON"
-        defs_w = []
-        unwrap_spec = self._code._unwrap_spec[-len(self._staticdefs):]
-        for i, (name, defaultval) in enumerate(self._staticdefs):
+        alldefs_w = {}
+        assert len(self._code._argnames) == len(self._code._unwrap_spec)
+        for name, spec in zip(self._code._argnames, self._code._unwrap_spec):
+            if name == '__kwonly__':
+                continue
+
+            defaultval = self._staticdefs.get(name, NO_DEFAULT)
+            w_def = Ellipsis
             if name.startswith('w_'):
-                assert defaultval is None, (
+                assert defaultval in (NO_DEFAULT, None), (
                     "%s: default value for '%s' can only be None, got %r; "
                     "use unwrap_spec(...=WrappedDefault(default))" % (
                     self._code.identifier, name, defaultval))
-                defs_w.append(None)
-            elif name != '__args__' and name != 'args_w':
-                spec = unwrap_spec[i]
-                if isinstance(defaultval, str) and spec not in [str]:
-                    defs_w.append(space.newbytes(defaultval))
-                else:
-                    defs_w.append(space.wrap(defaultval))
-        if self._code._unwrap_spec:
-            UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig[0])
-            if defs_w:
-                alldefs_w[-len(defs_w):] = defs_w
-            code = self._code
-            assert isinstance(code._unwrap_spec, (list, tuple))
-            assert isinstance(code._argnames, list)
-            assert len(code._unwrap_spec) == len(code._argnames)
-            for i in range(len(code._unwrap_spec)-1, -1, -1):
-                spec = code._unwrap_spec[i]
-                argname = code._argnames[i]
-                if isinstance(spec, tuple) and spec[0] is W_Root:
-                    assert False, "use WrappedDefault"
-                if isinstance(spec, WrappedDefault):
-                    default_value = spec.default_value
-                    if isinstance(default_value, str):
-                        w_default = space.newbytes(default_value)
-                    else:
-                        w_default = space.wrap(default_value)
-                    assert isinstance(w_default, W_Root)
-                    assert argname.startswith('w_')
-                    argname = argname[2:]
-                    j = self._code.sig[0].index(argname)
-                    assert alldefs_w[j] in (UNDEFINED, None)
-                    alldefs_w[j] = w_default
-            first_defined = 0
-            while (first_defined < len(alldefs_w) and
-                   alldefs_w[first_defined] is UNDEFINED):
-                first_defined += 1
-            defs_w = alldefs_w[first_defined:]
-            assert UNDEFINED not in defs_w
-        return defs_w
+                if defaultval is None:
+                    w_def = None
+
+            if isinstance(spec, tuple) and spec[0] is W_Root:
+                assert False, "use WrappedDefault"
+            elif isinstance(spec, WrappedDefault):
+                assert name.startswith('w_')
+                defaultval = spec.default_value
+                w_def = Ellipsis
+
+            if defaultval is not NO_DEFAULT:
+                if name != '__args__' and name != 'args_w':
+                    if w_def is Ellipsis:
+                        if isinstance(defaultval, str) and spec not in [str]:
+                            w_def = space.newbytes(defaultval)
+                        else:
+                            w_def = space.wrap(defaultval)
+                    if name.startswith('w_'):
+                        name = name[2:]
+                    alldefs_w[name] = w_def
+        #
+        # Here, 'alldefs_w' maps some argnames to their wrapped default
+        # value.  We return two lists:
+        #  - a list of defaults for positional arguments, which covers
+        #    some suffix of the sig.argnames list
+        #  - a list of pairs (w_name, w_def) for kwonly arguments
+        #
+        sig = self._code.sig
+        first_defined = 0
+        while (first_defined < len(sig.argnames) and
+               sig.argnames[first_defined] not in alldefs_w):
+            first_defined += 1
+        defs_w = [alldefs_w.pop(name) for name in sig.argnames[first_defined:]]
+
+        kw_defs_w = None
+        if alldefs_w:
+            kw_defs_w = []
+            for name, w_def in sorted(alldefs_w.items()):
+                assert name in sig.kwonlyargnames
+                w_name = space.newunicode(name.decode('utf-8'))
+                kw_defs_w.append((w_name, w_def))
+
+        return defs_w, kw_defs_w
 
     # lazy binding to space
 
@@ -1013,9 +1053,10 @@
     def build(cache, gateway):
         "NOT_RPYTHON"
         space = cache.space
-        defs = gateway._getdefaults(space) # needs to be implemented by subclass
+        defs_w, kw_defs_w = gateway._getdefaults(space)
         code = gateway._code
-        fn = FunctionWithFixedCode(space, code, None, defs, forcename=gateway.name)
+        fn = FunctionWithFixedCode(space, code, None, defs_w, kw_defs_w,
+                                   forcename=gateway.name)
         if not space.config.translating:
             fn.add_to_table()
         if gateway.as_classmethod:
@@ -1153,15 +1194,15 @@
             source = source[source.find('\n') + 1:].lstrip()
         assert source.startswith("def "), "can only transform functions"
         source = source[4:]
-        import __future__
-        if flags & __future__.CO_FUTURE_DIVISION:
-            prefix += "from __future__ import division\n"
-        if flags & __future__.CO_FUTURE_ABSOLUTE_IMPORT:
-            prefix += "from __future__ import absolute_import\n"
-        if flags & __future__.CO_FUTURE_PRINT_FUNCTION:
-            prefix += "from __future__ import print_function\n"
-        if flags & __future__.CO_FUTURE_UNICODE_LITERALS:
-            prefix += "from __future__ import unicode_literals\n"
+        # The following flags have no effect any more in app-level code
+        # (i.e. they are always on anyway), and have been removed:
+        #    CO_FUTURE_DIVISION
+        #    CO_FUTURE_ABSOLUTE_IMPORT
+        #    CO_FUTURE_PRINT_FUNCTION
+        #    CO_FUTURE_UNICODE_LITERALS
+        # Original code was, for each of these flags:
+        #    if flags & __future__.CO_xxx:
+        #        prefix += "from __future__ import yyy\n"
     p = source.find('(')
     assert p >= 0
     funcname = source[:p].strip()
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -6,7 +6,14 @@
 from rpython.rlib import jit
 
 
-class GeneratorIterator(W_Root):
+class GeneratorOrCoroutine(W_Root):
+    """XXX: move the common functionality here!"""
+
+    def descr_close(self):
+        raise NotImplementedError
+
+
+class GeneratorIterator(GeneratorOrCoroutine):
     "An iterator created by a generator."
     _immutable_fields_ = ['pycode']
 
@@ -333,7 +340,7 @@
                                       get_printable_location = get_printable_coroutine_location_genentry,
                                       name='coroutineentry')
 
-class Coroutine(W_Root):
+class Coroutine(GeneratorOrCoroutine):
     "A coroutine object."
     _immutable_fields_ = ['pycode']
 
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -19,6 +19,7 @@
         """ NOT_RPYTHON """
         Module.__init__(self, space, w_name)
         self.lazy = True
+        self.lazy_initial_values_w = {}
         self.__class__.buildloaders()
         self.loaders = self.loaders.copy()    # copy from the class to the inst
         self.submodules_w = []
@@ -57,22 +58,11 @@
             if not self.lazy and self.w_initialdict is None:
                 self.save_module_content_for_future_reload()
 
-    def save_module_content_for_future_reload(self, save_all=False):
-        # Because setdictvalue is unable to immediately load all attributes
-        # (due to an importlib bootstrapping problem), this method needs to be
-        # able to support saving the content of a module's dict without
-        # requiring that the entire dict already be loaded. To support that
-        # properly, when updating the dict, we must be careful to never
-        # overwrite the value of a key already in w_initialdict. (So as to avoid
-        # overriding the builtin value with a user-provided value)
-        if self.space.is_none(self.w_initialdict) or save_all:
-            self.w_initialdict = self.space.call_method(self.w_dict, 'copy')
-        else:
-            w_items = self.space.call_method(self.w_dict, 'items')
-            for w_item in self.space.iteriterable(w_items):
-                w_key, w_value = self.space.fixedview(w_item, expected_length=2)
-                if not self.space.contains_w(self.w_initialdict, w_key):
-                    self.space.setitem(self.w_initialdict, w_key, w_value)
+    def save_module_content_for_future_reload(self):
+        # Save the current dictionary in w_initialdict, for future
+        # reloads.  This forces the dictionary if needed.
+        w_dict = self.getdict(self.space)
+        self.w_initialdict = self.space.call_method(w_dict, 'copy')
 
     @classmethod
     def get_applevel_name(cls):
@@ -101,9 +91,13 @@
         return w_value
 
     def setdictvalue(self, space, attr, w_value):
-        if self.lazy:
-            self._load_lazily(space, attr)
-            self.save_module_content_for_future_reload()
+        if self.lazy and attr not in self.lazy_initial_values_w:
+            # in lazy mode, the first time an attribute changes,
+            # we save away the old (initial) value.  This allows
+            # a future getdict() call to build the correct
+            # self.w_initialdict, containing the initial value.
+            w_initial_value = self._load_lazily(space, attr)
+            self.lazy_initial_values_w[attr] = w_initial_value
         space.setitem_str(self.w_dict, attr, w_value)
         return True
 
@@ -137,13 +131,29 @@
 
     def getdict(self, space):
         if self.lazy:
-            for name in self.loaders:
-                w_value = self.get(name)
-                space.setitem(self.w_dict, space.new_interned_str(name), w_value)
-            self.lazy = False
-            self.save_module_content_for_future_reload()
+            self._force_lazy_dict_now()
         return self.w_dict
 
+    def _force_lazy_dict_now(self):
+        # Force the dictionary by calling all lazy loaders now.
+        # This also saves in self.w_initialdict a copy of all the
+        # initial values, including if they have already been
+        # modified by setdictvalue().
+        space = self.space
+        for name in self.loaders:
+            w_value = self.get(name)
+            space.setitem(self.w_dict, space.new_interned_str(name), w_value)
+        self.lazy = False
+        self.save_module_content_for_future_reload()
+        for key, w_initial_value in self.lazy_initial_values_w.items():
+            w_key = space.new_interned_str(key)
+            if w_initial_value is not None:
+                space.setitem(self.w_initialdict, w_key, w_initial_value)
+            else:
+                if space.finditem(self.w_initialdict, w_key) is not None:
+                    space.delitem(self.w_initialdict, w_key)
+        del self.lazy_initial_values_w
+
     def _cleanup_(self):
         self.getdict(self.space)
         self.w_initialdict = None
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -45,7 +45,7 @@
 
 # cpython_code_signature helper
 def cpython_code_signature(code):
-    "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
+    """Return a Signature instance."""
     argcount = code.co_argcount
     varnames = code.co_varnames
     if we_are_translated():
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,7 +1,7 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
-from rpython.rlib import jit
+from rpython.rlib import jit, rweakref
 from rpython.rlib.debug import make_sure_not_resized, check_nonneg
 from rpython.rlib.jit import hint
 from rpython.rlib.objectmodel import instantiate, specialize, we_are_translated
@@ -62,6 +62,8 @@
     __metaclass__ = extendabletype
 
     frame_finished_execution = False
+    f_generator_wref         = rweakref.dead_ref  # for generators/coroutines
+    f_generator_nowref       = None               # (only one of the two attrs)
     last_instr               = -1
     last_exception           = None
     f_backref                = jit.vref_None
@@ -238,14 +240,24 @@
             self.locals_cells_stack_w[index] = outer_func.closure[i]
             index += 1
 
+    def _is_generator_or_coroutine(self):
+        return (self.getcode().co_flags & (pycode.CO_COROUTINE |
+                                           pycode.CO_GENERATOR)) != 0
+
     def run(self):
         """Start this frame's execution."""
-        if self.getcode().co_flags & pycode.CO_COROUTINE:
-            from pypy.interpreter.generator import Coroutine
-            return self.space.wrap(Coroutine(self))
-        elif self.getcode().co_flags & pycode.CO_GENERATOR:
-            from pypy.interpreter.generator import GeneratorIterator
-            return self.space.wrap(GeneratorIterator(self))
+        if self._is_generator_or_coroutine():
+            if self.getcode().co_flags & pycode.CO_COROUTINE:
+                from pypy.interpreter.generator import Coroutine
+                gen = Coroutine(self)
+            else:
+                from pypy.interpreter.generator import GeneratorIterator
+                gen = GeneratorIterator(self)
+            if self.space.config.translation.rweakref:
+                self.f_generator_wref = rweakref.ref(gen)
+            else:
+                self.f_generator_nowref = gen
+            return self.space.wrap(gen)
         else:
             return self.execute_frame()
 
@@ -886,6 +898,37 @@
             frame = frame.f_backref()
         return None
 
+    def descr_clear(self, space):
+        # Clears a random subset of the attributes (e.g. some the fast
+        # locals, but not f_locals).  Also clears last_exception, which
+        # is not quite like CPython when it clears f_exc_* (however
+        # there might not be an observable difference).
+        if not self.frame_finished_execution:
+            if not self._is_generator_or_coroutine():
+                raise oefmt(space.w_RuntimeError,
+                            "cannot clear an executing frame")
+            if space.config.translation.rweakref:
+                gen = self.f_generator_wref()
+            else:
+                gen = self.f_generator_nowref
+            if gen is not None:
+                if gen.running:
+                    raise oefmt(space.w_RuntimeError,
+                                "cannot clear an executing frame")
+                # xxx CPython raises the RuntimeWarning "coroutine was never
+                # awaited" in this case too.  Does it make any sense?
+                gen.descr_close()
+
+        self.last_exception = None
+        debug = self.getdebug()
+        if debug is not None:
+            debug.w_f_trace = None
+
+        # clear the locals, including the cell/free vars, and the stack
+        for i in range(len(self.locals_cells_stack_w)):
+            self.locals_cells_stack_w[i] = None
+        self.valuestackdepth = 0
+
 # ____________________________________________________________
 
 def get_block_class(opname):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -96,7 +96,7 @@
             # Note that this case catches AttributeError!
             rstackovf.check_stack_overflow()
             next_instr = self.handle_asynchronous_error(ec,
-                self.space.w_RuntimeError,
+                self.space.w_RecursionError,
                 self.space.wrap("maximum recursion depth exceeded"))
         return next_instr
 
@@ -1288,15 +1288,15 @@
             for i in range(len(names_w) - 1, -1, -1):
                 space.setitem(w_ann, names_w[i], self.popvalue())
         defaultarguments = self.popvalues(posdefaults)
-        w_kw_defs = None
+        kw_defs_w = None
         if kwdefaults:
-            w_kw_defs = space.newdict(strdict=True)
-            for i in range(kwdefaults - 1, -1, -1):
-                w_name = self.popvalue()
-                w_def = self.popvalue()
-                space.setitem(w_kw_defs, w_def, w_name)
+            kw_defs_w = []
+            for i in range(kwdefaults):
+                w_defvalue = self.popvalue()
+                w_defname = self.popvalue()
+                kw_defs_w.append((w_defname, w_defvalue))
         fn = function.Function(space, codeobj, self.get_w_globals(), defaultarguments,
-                               w_kw_defs, freevars, w_ann, qualname=qualname)
+                               kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
     def MAKE_FUNCTION(self, oparg, next_instr):
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -39,6 +39,7 @@
 
     def scope_length(self):
         scopelen = len(self.argnames)
+        scopelen += len(self.kwonlyargnames)
         scopelen += self.has_vararg()
         scopelen += self.has_kwarg()
         return scopelen
@@ -68,18 +69,3 @@
         if not isinstance(other, Signature):
             return NotImplemented
         return not self == other
-
-
-    # make it look tuply for its use in the annotator
-
-    def __len__(self):
-        return 3
-
-    def __getitem__(self, i):
-        if i == 0:
-            return self.argnames
-        if i == 1:
-            return self.varargname
-        if i == 2:
-            return self.kwargname
-        raise IndexError
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -30,7 +30,7 @@
         assert sig.num_argnames() == 3
         assert sig.has_vararg()
         assert sig.has_kwarg()
-        assert sig.scope_length() == 5
+        assert sig.scope_length() == 6
         assert sig.getallvarnames() == ["a", "b", "c", "d", "kwonly", "c"]
 
     def test_eq(self):
@@ -47,13 +47,6 @@
         assert sig.find_argname("d") == -1
         assert sig.find_argname("kwonly") == 3
 
-    def test_tuply(self):
-        sig = Signature(["a", "b", "c"], "d", "e")
-        x, y, z = sig
-        assert x == ["a", "b", "c"]
-        assert y == "d"
-        assert z == "e"
-
 class dummy_wrapped_dict(dict):
     def __nonzero__(self):
         raise NotImplementedError
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -47,6 +47,12 @@
         code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace, "index"])
         assert code.signature() == Signature(["index"], None, None)
 
+        def f(space, __kwonly__, w_x):
+            pass
+        code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace,
+                                                   "kwonly", W_Root])
+        assert code.signature() == Signature([], kwonlyargnames=['x'])
+
 
     def test_call(self):
         def c(space, w_x, w_y, hello_w):
@@ -753,7 +759,7 @@
         @gateway.unwrap_spec(w_x = WrappedDefault(42), y=int)
         def g(space, w_x, y):
             never_called
-        py.test.raises(AssertionError, space.wrap, gateway.interp2app_temp(g))
+        py.test.raises(KeyError, space.wrap, gateway.interp2app_temp(g))
 
     def test_unwrap_spec_default_applevel_bug2(self):
         space = self.space
@@ -803,6 +809,80 @@
         w_res = space.call_args(w_g, args)
         assert space.eq_w(w_res, space.newbytes('foo'))
 
+    def test_unwrap_spec_kwonly(self):
+        space = self.space
+        def g(space, w_x, __kwonly__, w_y):
+            return space.sub(w_x, w_y)
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w = space.wrap
+        w1 = w(1)
+
+        for i in range(4):
+            a = argument.Arguments(space, [w1, w1, w1])
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+            py.test.raises(gateway.OperationError, space.call_function, w_g,
+                           *(i * (w1,)))
+
+        args = argument.Arguments(space, [w(1)],
+                                  w_starstararg = w({'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-9))
+        args = argument.Arguments(space, [],
+                                  w_starstararg = w({'x': 2, 'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-8))
+
+    def test_unwrap_spec_kwonly_default(self):
+        space = self.space
+        @gateway.unwrap_spec(w_x2=WrappedDefault(50), y2=int)
+        def g(space, w_x1, w_x2, __kwonly__, w_y1, y2=200):
+            return space.sub(space.sub(w_x1, w_x2),
+                             space.sub(w_y1, w(y2)))
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w = space.wrap
+        w1 = w(1)
+
+        for i in range(6):
+            py.test.raises(gateway.OperationError, space.call_function, w_g,
+                           *(i * (w1,)))
+
+        def expected(x1, x2=50, y1="missing", y2=200):
+            return (x1 - x2) - (y1 - y2)
+
+        def check(*args, **kwds):
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            w_res = space.call_args(w_g, a)
+            assert space.eq_w(w_res, w(expected(*args, **kwds)))
+
+            del kwds['y1']
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+
+            args += (1234,)
+            a = argument.Arguments(space, [], w_stararg = w(args),
+                                          w_starstararg = w(kwds))
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+
+        check(5,       y1=1234)
+        check(5, 1,    y1=1234)
+        check(5, x2=1, y1=1234)
+        check(5,       y1=1234, y2=343)
+        check(5, 1,    y1=1234, y2=343)
+        check(5, x2=1, y1=1234, y2=343)
+        check(x1=5,       y1=1234,       )
+        check(x1=5, x2=1, y1=1234,       )
+        check(x1=5,       y1=1234, y2=343)
+        check(x1=5, x2=1, y1=1234, y2=343)
+
+    def test_unwrap_spec_kwonly_default_2(self):
+        space = self.space
+        @gateway.unwrap_spec(w_x2=WrappedDefault(50))
+        def g(space, w_x2=None):
+            return w_x2
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_res = space.call_function(w_g)
+        assert space.eq_w(w_res, space.wrap(50))
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -220,22 +220,17 @@
         raises(RuntimeError, g.close)
 
     def test_close_on_collect(self):
-        ## we need to exec it, else it won't run on python2.4
-        d = {}
-        exec("""
         def f():
             try:
                 yield
             finally:
                 f.x = 42
-        """.strip(), d, d)
-
-        g = d['f']()
+        g = f()
         next(g)
         del g
         import gc
         gc.collect()
-        assert d['f'].x == 42
+        assert f.x == 42
 
     def test_generator_raises_typeerror(self):
         def f():
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -407,7 +407,7 @@
         def f(): f()
         try:
             f()
-        except RuntimeError as e:
+        except RecursionError as e:
             assert str(e) == "maximum recursion depth exceeded"
         else:
             assert 0, "should have raised!"
@@ -435,6 +435,20 @@
         assert X().f() == 42
         """
 
+    def test_kwonlyarg_required(self):
+        """
+        def f(*, a=5, b):
+            return (a, b)
+        assert f(b=10) == (5, 10)
+        assert f(a=7, b=12) == (7, 12)
+        raises(TypeError, f)
+        raises(TypeError, f, 1)
+        raises(TypeError, f, 1, 1)
+        raises(TypeError, f, a=1)
+        raises(TypeError, f, 1, a=1)
+        raises(TypeError, f, 1, b=1)
+        """
+
     def test_extended_unpacking_short(self):
         """
         class Seq:
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -165,7 +165,7 @@
         import sys
         import os
 
-        assert sys.__package__ is None
+        assert sys.__package__ == ''
         assert os.__package__ == ''
         assert not hasattr(type(sys)('foo'), '__package__')
 
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -545,3 +545,41 @@
         it = yield_raise()
         assert next(it) is KeyError
         assert next(it) is KeyError
+
+    def test_frame_clear(self):
+        import sys, gc, weakref
+        #
+        raises(RuntimeError, sys._getframe().clear)
+        def g():
+            yield 5
+            raises(RuntimeError, sys._getframe().clear)
+            yield 6
+        assert list(g()) == [5, 6]
+        #
+        class A:
+            pass
+        a1 = A(); a1ref = weakref.ref(a1)
+        a2 = A(); a2ref = weakref.ref(a2)
+        seen = []
+        def f():
+            local_a1 = a1
+            for loc in [5, 6, a2]:
+                try:
+                    yield sys._getframe()
+                finally:
+                    seen.append(42)
+                seen.append(43)
+        gen = f()
+        frame = next(gen)
+        a1 = a2 = None
+        gc.collect(); gc.collect()
+        assert a1ref() is not None
+        assert a2ref() is not None
+        assert seen == []
+        frame.clear()
+        assert seen == [42]
+        gc.collect(); gc.collect()
+        assert a1ref() is None, "locals not cleared"
+        assert a2ref() is None, "stack not cleared"
+        #
+        raises(StopIteration, next, gen)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -202,7 +202,8 @@
     name = func.__name__
     extra = ', '.join(extraargs)
     from pypy.interpreter import pycode
-    argnames, _, _ = pycode.cpython_code_signature(func.func_code)
+    sig = pycode.cpython_code_signature(func.func_code)
+    argnames = sig.argnames
     if use_closure:
         if argnames[1] == 'space':
             args = "closure, space, obj"
@@ -605,6 +606,7 @@
 PyFrame.typedef = TypeDef('frame',
     __reduce__ = interp2app(PyFrame.descr__reduce__),
     __setstate__ = interp2app(PyFrame.descr__setstate__),
+    clear = interp2app(PyFrame.descr_clear),
     f_builtins = GetSetProperty(PyFrame.fget_f_builtins),
     f_lineno = GetSetProperty(PyFrame.fget_f_lineno, PyFrame.fset_f_lineno),
     f_back = GetSetProperty(PyFrame.fget_f_back),
@@ -797,10 +799,15 @@
 )
 assert not GeneratorIterator.typedef.acceptable_as_base_class  # no __new__
 
+# TODO: to have the same distinction (Coroutine | Iterator) as in cpython 3.5,
+# a wrapper typedef with __anext__ has to be created, and __anext__ has to be
+# removed in coroutine
 Coroutine.typedef = TypeDef("coroutine",
     __repr__   = interp2app(Coroutine.descr__repr__),
     __reduce__   = interp2app(Coroutine.descr__reduce__),
     __setstate__ = interp2app(Coroutine.descr__setstate__),
+    __anext__   = interp2app(Coroutine.descr_next,
+                            descrmismatch='__anext__'),
     send       = interp2app(Coroutine.descr_send,
                             descrmismatch='send'),
     throw      = interp2app(Coroutine.descr_throw,
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -131,7 +131,7 @@
 
 @unwrap_spec(w_module=MixedModule)
 def save_module_content_for_future_reload(space, w_module):
-    w_module.save_module_content_for_future_reload(save_all=True)
+    w_module.save_module_content_for_future_reload()
 
 def set_code_callback(space, w_callable):
     cache = space.fromcache(CodeHookCache)
diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -118,9 +118,9 @@
     def test_object(self):
         ast = self.ast
         const = ast.Const(4)
-        assert const.value == 4
-        const.value = 5
-        assert const.value == 5
+        assert const.obj == 4
+        const.obj = 5
+        assert const.obj == 5
 
     def test_optional(self):
         mod = self.get_ast("x(32)", "eval")
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/pypy/module/_asyncio/test/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/pypy/module/_asyncio/test/test_asyncio.py
@@ -1,4 +1,5 @@
 class AppTestAsyncIO(object):
+    """These tests are based on the async-await syntax of Python 3.5."""
     
     spaceconfig = dict(usemodules=["select","_socket","thread","signal",
                                    "struct","_multiprocessing","array",
@@ -9,17 +10,71 @@
         # the problem occured at await asyncio.open_connection
         # after calling run_until_complete
         """
-        import encodings.idna
-        import asyncio
-        async def f():
-            reader, writer = await asyncio.open_connection('example.com', 80)
-        
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(f())
-        print("done with async loop")
+import encodings.idna
+import asyncio
+
+async def f():
+    reader, writer = await asyncio.open_connection('example.com', 80)
+    writer.close()
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(f())
+        """
+    
+    def test_async_for(self):
+        # tests if async for receives all stores values in the right order
+        # and if the correct methods __aiter__ and __anext__ get called
+        # and if the end results of run_until_complete are None (in a tuple)
+        """
+import asyncio
+
+class AsyncIter:
+    def __init__(self):
+        self._data = list(range(5))
+        self._index = 0
+    
+    async def __aiter__(self):
+        return self
+    
+    async def __anext__(self):
+        while self._index < 5:
+            await asyncio.sleep(1)
+            self._index += 1
+            return self._data[self._index-1]
+        raise StopAsyncIteration
+
+class Corotest(object):
+    def __init__(self):
+        self.res = "-"
+    
+    async def do_loop(self):
+        async for x in AsyncIter():
+            self.res += str(x)
+            self.res += "-"
+
+cor = Corotest()
+loop = asyncio.get_event_loop()
+futures = [asyncio.ensure_future(cor.do_loop()), asyncio.ensure_future(cor.do_loop())]
+taskres = loop.run_until_complete(asyncio.wait(futures))
+assert cor.res.count('0') == 2
+assert cor.res.count('1') == 2
+assert cor.res.count('2') == 2
+assert cor.res.count('3') == 2
+assert cor.res.count('4') == 2
+assert cor.res.find("0") < cor.res.find("1")
+assert cor.res.find("1") < cor.res.find("2")
+assert cor.res.find("2") < cor.res.find("3")
+assert cor.res.find("3") < cor.res.find("4")
+assert isinstance(taskres, tuple)
+assert len(taskres) == 2
+assert "result=None" in repr(taskres[0].pop())
+assert "result=None" in repr(taskres[0].pop())
         """
     
     def test_asynchronous_context_managers(self):
+        # it is important that "releasing lock A" happens before "holding lock B"
+        # or the other way around, but it is not allowed that both coroutines
+        # hold the lock at the same time
         """
 import encodings.idna
 import asyncio
@@ -44,5 +99,12 @@
 finally:
     loop.close()
 
-assert cor.res == "- coro 1: waiting for lock - coro 1: holding the lock - coro 2: waiting for lock - coro 1: releasing the lock - coro 2: holding the lock - coro 2: releasing the lock -"
+assert "coro 1: waiting for lock" in cor.res
+assert "coro 1: holding the lock" in cor.res
+assert "coro 1: releasing the lock" in cor.res
+assert "coro 2: waiting for lock" in cor.res
+assert "coro 2: holding the lock" in cor.res
+assert "coro 2: releasing the lock" in cor.res
+assert cor.res.find("coro 1: releasing the lock") < cor.res.find("coro 2: holding the lock") or \
+cor.res.find("coro 2: releasing the lock") < cor.res.find("coro 1: holding the lock")
         """
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -310,11 +310,15 @@
                             self.ctype.name, ct.name)
             #
             itemsize = ct.ctitem.size
-            if itemsize <= 0:
-                itemsize = 1
             with self as ptr1, w_other as ptr2:
                 diff = (rffi.cast(lltype.Signed, ptr1) -
-                        rffi.cast(lltype.Signed, ptr2)) // itemsize
+                        rffi.cast(lltype.Signed, ptr2))
+            if itemsize > 1:
+                if diff % itemsize:
+                    raise oefmt(space.w_ValueError,
+                        "pointer subtraction: the distance between the two "
+                        "pointers is not a multiple of the item size")
+                diff //= itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -257,6 +257,7 @@
     sandboxsafe=True)
 
 # split here for JIT backends that don't support floats/longlongs/etc.
+ at jit.dont_look_inside
 def is_nonnull_longdouble(cdata):
     return _is_nonnull_longdouble(read_raw_longdouble_data(cdata))
 def is_nonnull_float(cdata, size):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -576,6 +576,19 @@
     e = py.test.raises(TypeError, "q - a")
     assert str(e.value) == "cannot subtract cdata 'short *' and cdata 'int *'"
 
+def test_ptr_sub_unaligned():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    a = cast(BIntPtr, 1240)
+    for bi in range(1430, 1438):
+        b = cast(BIntPtr, bi)
+        if ((bi - 1240) % size_of_int()) == 0:
+            assert b - a == (bi - 1240) // size_of_int()
+            assert a - b == (1240 - bi) // size_of_int()
+        else:
+            py.test.raises(ValueError, "b - a")
+            py.test.raises(ValueError, "a - b")
+
 def test_cast_primitive_from_cdata():
     p = new_primitive_type("int")
     n = cast(p, cast(p, -42))
diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,10 +69,13 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
+        cls.w_fix_path = space.appexec([space.wrap(str(tmpdir))], """(path):
+            def fix_path(ignored=None):
+                import _cffi_backend     # force it to be initialized
+                import sys
+                if path not in sys.path:
+                    sys.path.insert(0, path)
+            return fix_path
         """)
 
     def teardown_method(self, meth):
@@ -86,17 +89,20 @@
 
 
     def test_constant_1(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
@@ -104,6 +110,7 @@
 
     def test_dlclose(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
@@ -115,17 +122,20 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
@@ -134,12 +144,14 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
+        self.fix_path()
         from re_py_subsrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
@@ -153,6 +165,7 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
@@ -163,24 +176,28 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == b"hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -79,7 +79,7 @@
 
 
 class AppTestRecompiler:
-    spaceconfig = dict(usemodules=['_cffi_backend', 'imp'])
+    spaceconfig = dict(usemodules=['_cffi_backend', 'imp', 'cpyext', 'struct'])
 
     def setup_class(cls):
         if cls.runappdirect:
diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -16,11 +16,10 @@
     @staticmethod
     def _compile_bootstrap_module(space, name, w_name, w_dict):
         """NOT_RPYTHON"""
-        ec = space.getexecutioncontext()
         with open(os.path.join(lib_python, 'importlib', name + '.py')) as fp:
             source = fp.read()
         pathname = "<frozen importlib.%s>" % name
-        code_w = ec.compiler.compile(source, pathname, 'exec', 0)
+        code_w = Module._cached_compile(space, source, pathname, 'exec', 0)
         space.setitem(w_dict, space.wrap('__name__'), w_name)
         space.setitem(w_dict, space.wrap('__builtins__'),
                       space.wrap(space.builtin))
@@ -43,6 +42,31 @@
 
         self.w_import = space.wrap(interp_import.import_with_frames_removed)
 
+    @staticmethod
+    def _cached_compile(space, source, *args):
+        from rpython.config.translationoption import CACHE_DIR
+        from pypy.module.marshal import interp_marshal
+
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_bootstrap')
+        try:
+            if space.config.translating:
+                raise IOError("don't use the cache when translating pypy")
+            with open(cachename, 'rb') as f:
+                previous = f.read(len(source) + 1)
+                if previous != source + '\x00':
+                    raise IOError("source changed")
+                w_bin = space.newbytes(f.read())
+                code_w = interp_marshal.loads(space, w_bin)
+        except IOError:
+            # must (re)compile the source
+            ec = space.getexecutioncontext()
+            code_w = ec.compiler.compile(source, *args)
+            w_bin = interp_marshal.dumps(space, code_w, space.wrap(2))
+            content = source + '\x00' + space.bytes_w(w_bin)
+            with open(cachename, 'wb') as f:
+                f.write(content)
+        return code_w
+
     def startup(self, space):
         """Copy our __import__ to builtins."""
         w_install = self.getdictvalue(space, '_install')
diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -10,10 +10,10 @@
 
     def setup_class(cls):
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__jitlog.1')))
-        cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
-        cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
+        cls.w_mark_header = cls.space.newbytes(jl.MARK_JITLOG_HEADER)
+        cls.w_version = cls.space.newbytes(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
-        cls.w_machine = cls.space.wrap(platform.machine())
+        cls.w_machine = cls.space.newbytes(platform.machine())
         cls.w_resops = cls.space.newdict()
         space = cls.space
         for key, value in opname.items():
@@ -48,5 +48,3 @@
                 assert opnum in self.resops
                 # the name must equal
                 assert self.resops[opnum] == opname
-
-
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
@@ -268,22 +268,26 @@
     min = 0;
     max = haystacksize;
 
-    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
+    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
         if (value < haystack[pos].uniseq) {
-            if (max == pos) break;
-            else max = pos;
+            if (max != pos) {
+                max = pos;
+                continue;
+            }
         }
         else if (value > haystack[pos].uniseq) {
-            if (min == pos) break;
-            else min = pos;
+            if (min != pos) {
+                min = pos;
+                continue;
+            }
         }
-        else
-            break;
+        break;
+    }
 
-        if (value == haystack[pos].uniseq)
-            return haystack[pos].code;
-        else
-            return DBCINV;
+    if (value == haystack[pos].uniseq) {
+        return haystack[pos].code;
+    }
+    return DBCINV;
 }
 #endif
 
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -327,7 +327,8 @@
             i += 1
             if ch == '"':
                 content_utf8 = builder.build()
-                content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
+                content_unicode = unicodehelper.decode_utf8(
+                    self.space, content_utf8, allow_surrogates=True)
                 self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.wrap(content_unicode)
@@ -374,7 +375,8 @@
                    # this point
         #
         uchr = runicode.code_to_unichr(val)     # may be a surrogate pair again
-        utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
+        utf8_ch = unicodehelper.encode_utf8(
+            self.space, uchr, allow_surrogates=True)
         builder.append(utf8_ch)
         return i
 
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -10,10 +10,10 @@
     assert dec.skip_whitespace(8) == len(s)
     dec.close()
 
-    
+
 
 class AppTest(object):
-    spaceconfig = {"objspace.usemodules._pypyjson": True}
+    spaceconfig = {"usemodules": ['_pypyjson']}
 
     def test_raise_on_bytes(self):
         import _pypyjson
@@ -40,7 +40,7 @@
         raises(ValueError, _pypyjson.loads, 'fa')
         raises(ValueError, _pypyjson.loads, 'f')
         raises(ValueError, _pypyjson.loads, 'falXX')
-        
+
 
     def test_decode_string(self):
         import _pypyjson
@@ -69,7 +69,7 @@
         import _pypyjson
         assert _pypyjson.loads(r'"\\"') == '\\'
         assert _pypyjson.loads(r'"\""') == '"'
-        assert _pypyjson.loads(r'"\/"') == '/'       

From pypy.commits at gmail.com  Mon Aug 22 04:32:11 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 22 Aug 2016 01:32:11 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: micro numpy translation
 fix (descr___buffer__ missing on W_Root),
 used wrap instead of newbytes (not present on TinyObjSpace)
Message-ID: <57bab88b.88cb1c0a.1eea1.0ee2@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86391:61d7052feef7
Date: 2016-08-22 10:31 +0200
http://bitbucket.org/pypy/pypy/changeset/61d7052feef7/

Log:	micro numpy translation fix (descr___buffer__ missing on W_Root),
	used wrap instead of newbytes (not present on TinyObjSpace)

diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -10,10 +10,10 @@
 
     def setup_class(cls):
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__jitlog.1')))
-        cls.w_mark_header = cls.space.newbytes(jl.MARK_JITLOG_HEADER)
-        cls.w_version = cls.space.newbytes(jl.JITLOG_VERSION_16BIT_LE)
+        cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
+        cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
-        cls.w_machine = cls.space.newbytes(platform.machine())
+        cls.w_machine = cls.space.wrap(platform.machine())
         cls.w_resops = cls.space.newdict()
         space = cls.space
         for key, value in opname.items():
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -19,7 +19,6 @@
     UserDelAction)
 from pypy.interpreter.pyframe import PyFrame
 
-
 class BogusBytecode(Exception):
     pass
 
@@ -383,6 +382,9 @@
         # XXX even the hacks have hacks
         if s == 'size': # used in _array() but never called by tests
             return IntObject(0)
+        if s == '__buffer__':
+            # descr___buffer__ does not exist on W_Root
+            return self.w_None
         return getattr(w_obj, 'descr_' + s)(self, *args)
 
     @specialize.arg(1)
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -468,7 +468,7 @@
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_buffer = space.call_method(w_buffer, '__buffer__', 0)
+        w_buffer = space.call_method(w_buffer, '__buffer__', space.wrap(0))
         buf = _getbuffer(space, w_buffer)
 
     ts = buf.getlength()

From pypy.commits at gmail.com  Mon Aug 22 04:33:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 01:33:05 -0700 (PDT)
Subject: [pypy-commit] pypy default: Test and fix for rposix.fdopendir()
Message-ID: <57bab8c1.4219c20a.c8f99.ede2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86392:43568988afe8
Date: 2016-08-22 10:05 +0200
http://bitbucket.org/pypy/pypy/changeset/43568988afe8/

Log:	Test and fix for rposix.fdopendir()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -620,6 +620,8 @@
         [rffi.CCHARP], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
     c_fdopendir = external('fdopendir',
         [rffi.INT], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
+    c_rewinddir = external('rewinddir',
+        [DIRP], lltype.Void, releasegil=False)
     # XXX macro=True is hack to make sure we get the correct kind of
     # dirent struct (which depends on defines)
     c_readdir = external('readdir', [DIRP], DIRENTP,
@@ -629,7 +631,7 @@
 else:
     dirent_config = {}
 
-def _listdir(dirp):
+def _listdir(dirp, rewind=False):
     result = []
     while True:
         direntp = c_readdir(dirp)
@@ -640,6 +642,8 @@
         name = rffi.charp2str(namep)
         if name != '.' and name != '..':
             result.append(name)
+    if rewind:
+        c_rewinddir(dirp)
     c_closedir(dirp)
     if error:
         raise OSError(error, "readdir failed")
@@ -650,12 +654,16 @@
     Like listdir(), except that the directory is specified as an open
     file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.
+    Note: fdlistdir() closes the file descriptor.  To emulate the
+    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+    file descriptor.
     """
     dirp = c_fdopendir(dirfd)
     if not dirp:
-        raise OSError(get_saved_errno(), "opendir failed")
-    return _listdir(dirp)
+        error = get_saved_errno()
+        c_close(dirfd)
+        raise OSError(error, "opendir failed")
+    return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)
@@ -1799,7 +1807,7 @@
                   'fcntl.h'],
     )
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
-            fdopendir fpathconf fstat fstatat fstatvfs ftruncate
+            fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown
             lstat lutimes mkdirat mkfifoat mknodat openat readlinkat renameat
             symlinkat unlinkat utimensat""".split():
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -552,6 +552,14 @@
     # Note: fdlistdir() always closes dirfd
     assert result == ['file']
 
+ at rposix_requires('fdlistdir')
+def test_fdlistdir_rewinddir(tmpdir):
+    tmpdir.join('file').write('text')
+    dirfd = os.open(str(tmpdir), os.O_RDONLY)
+    result1 = rposix.fdlistdir(os.dup(dirfd))
+    result2 = rposix.fdlistdir(dirfd)
+    assert result1 == result2 == ['file']
+
 @rposix_requires('symlinkat')
 def test_symlinkat(tmpdir):
     tmpdir.join('file').write('text')

From pypy.commits at gmail.com  Mon Aug 22 04:33:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 01:33:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix HAVE_FDOPENDIR,
 which was always False
Message-ID: <57bab8c3.eeb8c20a.7e972.f712@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86393:15a11c0744f3
Date: 2016-08-22 10:10 +0200
http://bitbucket.org/pypy/pypy/changeset/15a11c0744f3/

Log:	Fix HAVE_FDOPENDIR, which was always False

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1800,14 +1800,9 @@
 # Support for f... and ...at families of POSIX functions
 
 class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes=['sys/stat.h',
-                  'sys/time.h',
-                  'unistd.h',
-                  'fcntl.h'],
-    )
+    _compilation_info_ = eci
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
-            fpathconf fstat fstatat fstatvfs ftruncate
+            fdopendir fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown
             lstat lutimes mkdirat mkfifoat mknodat openat readlinkat renameat
             symlinkat unlinkat utimensat""".split():

From pypy.commits at gmail.com  Mon Aug 22 04:33:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 01:33:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57bab8c4.6974c20a.af6a3.f2f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86394:fb2a0382b7eb
Date: 2016-08-22 10:11 +0200
http://bitbucket.org/pypy/pypy/changeset/fb2a0382b7eb/

Log:	hg merge default

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -620,15 +620,18 @@
         [rffi.CCHARP], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
     c_fdopendir = external('fdopendir',
         [rffi.INT], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
+    c_rewinddir = external('rewinddir',
+        [DIRP], lltype.Void, releasegil=False)
     # XXX macro=True is hack to make sure we get the correct kind of
     # dirent struct (which depends on defines)
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
+    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
 else:
     dirent_config = {}
 
-def _listdir(dirp):
+def _listdir(dirp, rewind=False):
     result = []
     while True:
         direntp = c_readdir(dirp)
@@ -639,6 +642,8 @@
         name = rffi.charp2str(namep)
         if name != '.' and name != '..':
             result.append(name)
+    if rewind:
+        c_rewinddir(dirp)
     c_closedir(dirp)
     if error:
         raise OSError(error, "readdir failed")
@@ -649,12 +654,16 @@
     Like listdir(), except that the directory is specified as an open
     file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.
+    Note: fdlistdir() closes the file descriptor.  To emulate the
+    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+    file descriptor.
     """
     dirp = c_fdopendir(dirfd)
     if not dirp:
-        raise OSError(get_saved_errno(), "opendir failed")
-    return _listdir(dirp)
+        error = get_saved_errno()
+        c_close(dirfd)
+        raise OSError(error, "opendir failed")
+    return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)
@@ -1791,12 +1800,7 @@
 # Support for f... and ...at families of POSIX functions
 
 class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes=['sys/stat.h',
-                  'sys/time.h',
-                  'unistd.h',
-                  'fcntl.h'],
-    )
+    _compilation_info_ = eci
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
             fdopendir fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -42,9 +42,9 @@
     return rffi.charp2str(namep)
 
 DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0)
-DT_REG = rposix.dirent_config.get('DT_REG', -1)
-DT_DIR = rposix.dirent_config.get('DT_DIR', -1)
-DT_LNK = rposix.dirent_config.get('DT_LNK', -1)
+DT_REG = rposix.dirent_config.get('DT_REG', 255)
+DT_DIR = rposix.dirent_config.get('DT_DIR', 255)
+DT_LNK = rposix.dirent_config.get('DT_LNK', 255)
 
 def get_known_type(direntp):
     if rposix.HAVE_D_TYPE:
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -552,6 +552,14 @@
     # Note: fdlistdir() always closes dirfd
     assert result == ['file']
 
+ at rposix_requires('fdlistdir')
+def test_fdlistdir_rewinddir(tmpdir):
+    tmpdir.join('file').write('text')
+    dirfd = os.open(str(tmpdir), os.O_RDONLY)
+    result1 = rposix.fdlistdir(os.dup(dirfd))
+    result2 = rposix.fdlistdir(dirfd)
+    assert result1 == result2 == ['file']
+
 @rposix_requires('symlinkat')
 def test_symlinkat(tmpdir):
     tmpdir.join('file').write('text')

From pypy.commits at gmail.com  Mon Aug 22 04:33:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 01:33:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57bab8ca.6211c20a.3ee9e.ee48@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86397:9629ba2a4200
Date: 2016-08-22 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/9629ba2a4200/

Log:	hg merge py3k

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -838,8 +838,7 @@
             raise wrap_oserror(space, e)
 
 
- at unwrap_spec(w_path=WrappedDefault(u"."))
-def listdir(space, w_path):
+def listdir(space, w_path=None):
     """listdir(path='.') -> list_of_filenames
 
 Return a list containing the names of the files in the directory.
@@ -852,6 +851,8 @@
 On some platforms, path may also be specified as an open file descriptor;
   the file descriptor must refer to a directory.
   If this functionality is unavailable, using it raises NotImplementedError."""
+    if space.is_none(w_path):
+        w_path = space.newunicode(u".")
     if space.isinstance_w(w_path, space.w_bytes):
         dirname = space.str0_w(w_path)
         try:
@@ -869,9 +870,9 @@
                 "listdir: illegal type for path argument")
         fd = unwrap_fd(space, w_path, "string, bytes or integer")
         try:
-            result = rposix.fdlistdir(fd)
+            result = rposix.fdlistdir(os.dup(fd))
         except OSError as e:
-            raise wrap_oserror2(space, e, w_path)
+            raise wrap_oserror(space, e)
     else:
         dirname = FileEncoder(space, w_path)
         try:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -313,7 +313,7 @@
 
     def test_listdir_default(self):
         posix = self.posix
-        assert posix.listdir() == posix.listdir('.')
+        assert posix.listdir() == posix.listdir('.') == posix.listdir(None)
 
     def test_listdir_bytes(self):
         import sys
@@ -325,6 +325,17 @@
         expected = b'caf%E9' if sys.platform == 'darwin' else b'caf\xe9'
         assert expected in result
 
+    def test_fdlistdir(self):
+        posix = self.posix
+        dirfd = posix.open('.', posix.O_RDONLY)
+        lst1 = posix.listdir(dirfd)   # does not close dirfd
+        lst2 = posix.listdir('.')
+        assert lst1 == lst2
+        #
+        lst3 = posix.listdir(dirfd)   # rewinddir() was used
+        assert lst3 == lst1
+        posix.close(dirfd)
+
     def test_undecodable_filename(self):
         import sys
         posix = self.posix
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -620,15 +620,18 @@
         [rffi.CCHARP], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
     c_fdopendir = external('fdopendir',
         [rffi.INT], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
+    c_rewinddir = external('rewinddir',
+        [DIRP], lltype.Void, releasegil=False)
     # XXX macro=True is hack to make sure we get the correct kind of
     # dirent struct (which depends on defines)
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
+    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
 else:
     dirent_config = {}
 
-def _listdir(dirp):
+def _listdir(dirp, rewind=False):
     result = []
     while True:
         direntp = c_readdir(dirp)
@@ -639,6 +642,8 @@
         name = rffi.charp2str(namep)
         if name != '.' and name != '..':
             result.append(name)
+    if rewind:
+        c_rewinddir(dirp)
     c_closedir(dirp)
     if error:
         raise OSError(error, "readdir failed")
@@ -649,12 +654,16 @@
     Like listdir(), except that the directory is specified as an open
     file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.
+    Note: fdlistdir() closes the file descriptor.  To emulate the
+    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+    file descriptor.
     """
     dirp = c_fdopendir(dirfd)
     if not dirp:
-        raise OSError(get_saved_errno(), "opendir failed")
-    return _listdir(dirp)
+        error = get_saved_errno()
+        c_close(dirfd)
+        raise OSError(error, "opendir failed")
+    return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)
@@ -1791,12 +1800,7 @@
 # Support for f... and ...at families of POSIX functions
 
 class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes=['sys/stat.h',
-                  'sys/time.h',
-                  'unistd.h',
-                  'fcntl.h'],
-    )
+    _compilation_info_ = eci
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
             fdopendir fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -1,6 +1,6 @@
 from rpython.rlib import rposix
 from rpython.rlib.objectmodel import specialize
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 
 @specialize.argtype(0)
@@ -17,21 +17,22 @@
 def closedir(dirp):
     rposix.c_closedir(dirp)
 
+NULL_DIRP = lltype.nullptr(rposix.DIRENT)
+
 def nextentry(dirp):
     """Read the next entry and returns an opaque object.
     Use the methods has_xxx() and get_xxx() to read from that
     opaque object.  The opaque object is valid until the next
     time nextentry() or closedir() is called.  This may raise
-    StopIteration, or OSError.  Note that this doesn't filter
-    out the "." and ".." entries.
+    OSError, or return a NULL pointer when exhausted.  Note
+    that this doesn't filter out the "." and ".." entries.
     """
     direntp = rposix.c_readdir(dirp)
     if direntp:
-        return direntp
-    error = rposix.get_saved_errno()
-    if error:
-        raise OSError(error, "readdir failed")
-    raise StopIteration
+        error = rposix.get_saved_errno()
+        if error:
+            raise OSError(error, "readdir failed")
+    return direntp
 
 def has_name_bytes(direntp):
     return True
@@ -40,20 +41,12 @@
     namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
     return rffi.charp2str(namep)
 
-DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', None)
-DT_REG = rposix.dirent_config.get('DT_REG', None)
-DT_DIR = rposix.dirent_config.get('DT_DIR', None)
-DT_LNK = rposix.dirent_config.get('DT_LNK', None)
+DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0)
+DT_REG = rposix.dirent_config.get('DT_REG', 255)
+DT_DIR = rposix.dirent_config.get('DT_DIR', 255)
+DT_LNK = rposix.dirent_config.get('DT_LNK', 255)
 
-def has_type(direntp):
-    return (DT_UNKNOWN is not None and
-            rffi.getintfield(direntp, 'c_d_type') != DT_UNKNOWN)
-
-def type_is_regular(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_REG
-
-def type_is_dir(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_DIR
-
-def type_is_link(direntp):
-    return rffi.getintfield(direntp, 'c_d_type') == DT_LNK
+def get_known_type(direntp):
+    if rposix.HAVE_D_TYPE:
+        return rffi.getintfield(direntp, 'c_d_type')
+    return DT_UNKNOWN
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -552,6 +552,14 @@
     # Note: fdlistdir() always closes dirfd
     assert result == ['file']
 
+ at rposix_requires('fdlistdir')
+def test_fdlistdir_rewinddir(tmpdir):
+    tmpdir.join('file').write('text')
+    dirfd = os.open(str(tmpdir), os.O_RDONLY)
+    result1 = rposix.fdlistdir(os.dup(dirfd))
+    result2 = rposix.fdlistdir(dirfd)
+    assert result1 == result2 == ['file']
+
 @rposix_requires('symlinkat')
 def test_symlinkat(tmpdir):
     tmpdir.join('file').write('text')
diff --git a/rpython/rlib/test/test_rposix_scandir.py b/rpython/rlib/test/test_rposix_scandir.py
--- a/rpython/rlib/test/test_rposix_scandir.py
+++ b/rpython/rlib/test/test_rposix_scandir.py
@@ -10,9 +10,8 @@
         scan = rposix_scandir.opendir('/')
         found = []
         while True:
-            try:
-                p = rposix_scandir.nextentry(scan)
-            except StopIteration:
+            p = rposix_scandir.nextentry(scan)
+            if not p:
                 break
             assert rposix_scandir.has_name_bytes(p)
             found.append(rposix_scandir.get_name_bytes(p))

From pypy.commits at gmail.com  Mon Aug 22 04:33:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 01:33:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Test and fix for os.listdir(fd)
Message-ID: <57bab8c6.88cb1c0a.1eea1.0f45@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86395:daf7eb91064e
Date: 2016-08-22 10:15 +0200
http://bitbucket.org/pypy/pypy/changeset/daf7eb91064e/

Log:	Test and fix for os.listdir(fd)

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -872,9 +872,9 @@
                 "argument should be string, bytes or integer, not %T", w_path)
         fd = unwrap_fd(space, w_path)
         try:
-            result = rposix.fdlistdir(fd)
+            result = rposix.fdlistdir(os.dup(fd))
         except OSError as e:
-            raise wrap_oserror2(space, e, w_path)
+            raise wrap_oserror(space, e)
     else:
         dirname = FileEncoder(space, w_path)
         try:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -325,6 +325,17 @@
         expected = b'caf%E9' if sys.platform == 'darwin' else b'caf\xe9'
         assert expected in result
 
+    def test_fdlistdir(self):
+        posix = self.posix
+        dirfd = posix.open('.', posix.O_RDONLY)
+        lst1 = posix.listdir(dirfd)   # does not close dirfd
+        lst2 = posix.listdir('.')
+        assert lst1 == lst2
+        #
+        lst3 = posix.listdir(dirfd)   # rewinddir() was used
+        assert lst3 == lst1
+        posix.close(dirfd)
+
     def test_undecodable_filename(self):
         import sys
         posix = self.posix

From pypy.commits at gmail.com  Mon Aug 22 04:33:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 01:33:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: hg merge py3.5
Message-ID: <57bab8cc.2916c20a.ed206.ef89@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86398:1abeaf52ade5
Date: 2016-08-22 10:30 +0200
http://bitbucket.org/pypy/pypy/changeset/1abeaf52ade5/

Log:	hg merge py3.5

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -838,8 +838,7 @@
             raise wrap_oserror(space, e)
 
 
- at unwrap_spec(w_path=WrappedDefault(u"."))
-def listdir(space, w_path):
+def listdir(space, w_path=None):
     """listdir(path='.') -> list_of_filenames
 
 Return a list containing the names of the files in the directory.
@@ -852,6 +851,8 @@
 On some platforms, path may also be specified as an open file descriptor;
   the file descriptor must refer to a directory.
   If this functionality is unavailable, using it raises NotImplementedError."""
+    if space.is_none(w_path):
+        w_path = space.newunicode(u".")
     if space.isinstance_w(w_path, space.w_bytes):
         dirname = space.str0_w(w_path)
         try:
@@ -869,9 +870,9 @@
                 "listdir: illegal type for path argument")
         fd = unwrap_fd(space, w_path, "string, bytes or integer")
         try:
-            result = rposix.fdlistdir(fd)
+            result = rposix.fdlistdir(os.dup(fd))
         except OSError as e:
-            raise wrap_oserror2(space, e, w_path)
+            raise wrap_oserror(space, e)
     else:
         dirname = FileEncoder(space, w_path)
         try:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -313,7 +313,7 @@
 
     def test_listdir_default(self):
         posix = self.posix
-        assert posix.listdir() == posix.listdir('.')
+        assert posix.listdir() == posix.listdir('.') == posix.listdir(None)
 
     def test_listdir_bytes(self):
         import sys
@@ -325,6 +325,17 @@
         expected = b'caf%E9' if sys.platform == 'darwin' else b'caf\xe9'
         assert expected in result
 
+    def test_fdlistdir(self):
+        posix = self.posix
+        dirfd = posix.open('.', posix.O_RDONLY)
+        lst1 = posix.listdir(dirfd)   # does not close dirfd
+        lst2 = posix.listdir('.')
+        assert lst1 == lst2
+        #
+        lst3 = posix.listdir(dirfd)   # rewinddir() was used
+        assert lst3 == lst1
+        posix.close(dirfd)
+
     def test_undecodable_filename(self):
         import sys
         posix = self.posix
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -620,6 +620,8 @@
         [rffi.CCHARP], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
     c_fdopendir = external('fdopendir',
         [rffi.INT], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
+    c_rewinddir = external('rewinddir',
+        [DIRP], lltype.Void, releasegil=False)
     # XXX macro=True is hack to make sure we get the correct kind of
     # dirent struct (which depends on defines)
     c_readdir = external('readdir', [DIRP], DIRENTP,
@@ -629,7 +631,7 @@
 else:
     dirent_config = {}
 
-def _listdir(dirp):
+def _listdir(dirp, rewind=False):
     result = []
     while True:
         direntp = c_readdir(dirp)
@@ -640,6 +642,8 @@
         name = rffi.charp2str(namep)
         if name != '.' and name != '..':
             result.append(name)
+    if rewind:
+        c_rewinddir(dirp)
     c_closedir(dirp)
     if error:
         raise OSError(error, "readdir failed")
@@ -650,12 +654,16 @@
     Like listdir(), except that the directory is specified as an open
     file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.
+    Note: fdlistdir() closes the file descriptor.  To emulate the
+    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+    file descriptor.
     """
     dirp = c_fdopendir(dirfd)
     if not dirp:
-        raise OSError(get_saved_errno(), "opendir failed")
-    return _listdir(dirp)
+        error = get_saved_errno()
+        c_close(dirfd)
+        raise OSError(error, "opendir failed")
+    return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)
@@ -1792,12 +1800,7 @@
 # Support for f... and ...at families of POSIX functions
 
 class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes=['sys/stat.h',
-                  'sys/time.h',
-                  'unistd.h',
-                  'fcntl.h'],
-    )
+    _compilation_info_ = eci
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
             fdopendir fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -552,6 +552,14 @@
     # Note: fdlistdir() always closes dirfd
     assert result == ['file']
 
+ at rposix_requires('fdlistdir')
+def test_fdlistdir_rewinddir(tmpdir):
+    tmpdir.join('file').write('text')
+    dirfd = os.open(str(tmpdir), os.O_RDONLY)
+    result1 = rposix.fdlistdir(os.dup(dirfd))
+    result2 = rposix.fdlistdir(dirfd)
+    assert result1 == result2 == ['file']
+
 @rposix_requires('symlinkat')
 def test_symlinkat(tmpdir):
     tmpdir.join('file').write('text')

From pypy.commits at gmail.com  Mon Aug 22 04:33:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 01:33:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: os.listdir(None)
Message-ID: <57bab8c8.8628c20a.67fff.eaec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86396:3951644b86e8
Date: 2016-08-22 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/3951644b86e8/

Log:	os.listdir(None)

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -838,8 +838,7 @@
             raise wrap_oserror(space, e)
 
 
- at unwrap_spec(w_path=WrappedDefault(u"."))
-def listdir(space, w_path):
+def listdir(space, w_path=None):
     """listdir(path='.') -> list_of_filenames
 
 Return a list containing the names of the files in the directory.
@@ -852,6 +851,8 @@
 On some platforms, path may also be specified as an open file descriptor;
   the file descriptor must refer to a directory.
   If this functionality is unavailable, using it raises NotImplementedError."""
+    if space.is_none(w_path):
+        w_path = space.newunicode(u".")
     if space.isinstance_w(w_path, space.w_bytes):
         dirname = space.str0_w(w_path)
         try:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -313,7 +313,7 @@
 
     def test_listdir_default(self):
         posix = self.posix
-        assert posix.listdir() == posix.listdir('.')
+        assert posix.listdir() == posix.listdir('.') == posix.listdir(None)
 
     def test_listdir_bytes(self):
         import sys

From pypy.commits at gmail.com  Mon Aug 22 04:33:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 01:33:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: Fix os.scandir(None),
 and os.scandir(dirfd) does not work on cpython either
Message-ID: <57bab8cd.a710c20a.b630c.e62e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86399:bcb73d504b55
Date: 2016-08-22 10:32 +0200
http://bitbucket.org/pypy/pypy/changeset/bcb73d504b55/

Log:	Fix os.scandir(None), and os.scandir(dirfd) does not work on cpython
	either

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -11,21 +11,16 @@
 from pypy.module.posix.interp_posix import unwrap_fd, build_stat_result
 
 
- at unwrap_spec(w_path=WrappedDefault(u"."))
-def scandir(space, w_path):
+def scandir(space, w_path=None):
     "scandir(path='.') -> iterator of DirEntry objects for given path"
 
+    if space.is_none(w_path):
+        w_path = space.newunicode(u".")
     if space.isinstance_w(w_path, space.w_bytes):
         path_bytes = space.str0_w(w_path)
         result_is_bytes = True
     else:
-        try:
-            path_bytes = space.fsencode_w(w_path)
-        except OperationError as operr:
-            if operr.async(space):
-                raise
-            fd = unwrap_fd(space, w_path, "string, bytes or integer")
-            XXXX
+        path_bytes = space.fsencode_w(w_path)
         result_is_bytes = False
 
     try:
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -61,6 +61,10 @@
         assert type(d.name) is str
         assert type(d.path) is str
         assert d.path == './' + d.name
+        d = next(posix.scandir(None))
+        assert type(d.name) is str
+        assert type(d.path) is str
+        assert d.path == './' + d.name
         d = next(posix.scandir(u'.'))
         assert type(d.name) is str
         assert type(d.path) is str
@@ -149,3 +153,7 @@
         raises(OSError, d.is_file)
         raises(OSError, d.is_dir)
         assert d.is_symlink()
+
+    def test_fdopendir_unsupported(self):
+        posix = self.posix
+        raises(TypeError, posix.scandir, 1234)

From pypy.commits at gmail.com  Mon Aug 22 04:58:40 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 22 Aug 2016 01:58:40 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: remove gcc warning/error
 on buildbot for cpyext test
Message-ID: <57babec0.44ce1c0a.325f4.0e48@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86400:73e384ef0911
Date: 2016-08-22 10:57 +0200
http://bitbucket.org/pypy/pypy/changeset/73e384ef0911/

Log:	remove gcc warning/error on buildbot for cpyext test

diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -15,9 +15,10 @@
 
 /* initialize the array with integers 0...length */
 void initialize_MyArray(MyArray* a, long length){
+    int i;
     a->length = length;
     a->arr = (int*)malloc(length * sizeof(int));
-    for(int i=0; i<length; i++){
+    for(i=0; i<length; i++){
         a->arr[i] = i;
     }
 }

From pypy.commits at gmail.com  Mon Aug 22 05:02:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 02:02:29 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: progress
Message-ID: <57babfa5.262ec20a.11a57.f8c2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5679:b7f1bca916c9
Date: 2016-08-22 11:02 +0200
http://bitbucket.org/pypy/extradoc/changeset/b7f1bca916c9/

Log:	progress

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -11,8 +11,6 @@
   We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
   Matti's opinion on that would be great!
 
-* os.scandir() (arigo)
-
 
 Finished
 --------
@@ -55,7 +53,7 @@
 * A new RecursionError exception is now raised when maximum recursion
   depth is reached. (DONE)
 
-* The new os.scandir() function
+* The new os.scandir() function (POSIX-DONE, missing Win32)
 
 * Newly created file descriptors are non-inheritable (PEP 446)
 

From pypy.commits at gmail.com  Mon Aug 22 05:02:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 02:02:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: ready for merge
Message-ID: <57babfb1.cb7f1c0a.49c70.1dcd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86401:c85430b9119f
Date: 2016-08-22 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/c85430b9119f/

Log:	ready for merge


From pypy.commits at gmail.com  Mon Aug 22 05:02:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 02:02:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3.5-scandir
Message-ID: <57babfb2.497bc20a.7638e.f66f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86402:8927e9200840
Date: 2016-08-22 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/8927e9200840/

Log:	hg merge py3.5-scandir

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -1,30 +1,26 @@
+import stat
+from errno import ENOENT
 from rpython.rlib import rgc
-from rpython.rlib import rposix_scandir
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib import rposix, rposix_scandir, rposix_stat
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, interp2app
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror2
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.baseobjspace import W_Root
 
-from pypy.module.posix.interp_posix import unwrap_fd
+from pypy.module.posix.interp_posix import unwrap_fd, build_stat_result
 
 
- at unwrap_spec(w_path=WrappedDefault(u"."))
-def scandir(space, w_path):
+def scandir(space, w_path=None):
     "scandir(path='.') -> iterator of DirEntry objects for given path"
 
+    if space.is_none(w_path):
+        w_path = space.newunicode(u".")
     if space.isinstance_w(w_path, space.w_bytes):
         path_bytes = space.str0_w(w_path)
         result_is_bytes = True
     else:
-        try:
-            path_bytes = space.fsencode_w(w_path)
-        except OperationError as operr:
-            if operr.async(space):
-                raise
-            fd = unwrap_fd(space, w_path, "string, bytes or integer")
-            XXXX
+        path_bytes = space.fsencode_w(w_path)
         result_is_bytes = False
 
     try:
@@ -37,13 +33,20 @@
     w_path_prefix = space.newbytes(path_prefix)
     if not result_is_bytes:
         w_path_prefix = space.fsdecode(w_path_prefix)
-    return W_ScandirIterator(space, dirp, w_path_prefix, result_is_bytes)
+    if rposix.HAVE_FSTATAT:
+        dirfd = rposix.c_dirfd(dirp)
+    else:
+        dirfd = -1
+    return W_ScandirIterator(space, dirp, dirfd, w_path_prefix, result_is_bytes)
 
 
 class W_ScandirIterator(W_Root):
-    def __init__(self, space, dirp, w_path_prefix, result_is_bytes):
+    _in_next = False
+
+    def __init__(self, space, dirp, dirfd, w_path_prefix, result_is_bytes):
         self.space = space
         self.dirp = dirp
+        self.dirfd = dirfd
         self.w_path_prefix = w_path_prefix
         self.result_is_bytes = result_is_bytes
 
@@ -58,7 +61,8 @@
     def fail(self, err=None):
         dirp = self.dirp
         if dirp:
-            self.dirp = lltype.nullptr(lltype.typeOf(dirp).TO)
+            self.dirfd = -1
+            self.dirp = rposix_scandir.NULL_DIRP
             rposix_scandir.closedir(dirp)
         if err is None:
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
@@ -66,29 +70,31 @@
             raise err
 
     def next_w(self):
-        # XXX not safe against being called on several threads for
-        # the same object, but I think that CPython has the same problem
         if not self.dirp:
             self.fail()
-        #
-        space = self.space
-        while True:
-            try:
-                entry = rposix_scandir.nextentry(self.dirp)
-            except StopIteration:
-                self.fail()
-            except OSError as e:
-                self.fail(wrap_oserror(space, e))
-            assert rposix_scandir.has_name_bytes(entry)
-            name = rposix_scandir.get_name_bytes(entry)
-            if name != '.' and name != '..':
-                break
-        #
-        w_name = space.newbytes(name)
-        result_is_bytes = self.result_is_bytes
-        if not result_is_bytes:
-            w_name = space.fsdecode(w_name)
-        direntry = W_DirEntry(w_name, self.w_path_prefix, result_is_bytes)
+        if self._in_next:
+            self.fail(oefmt(self.space.w_RuntimeError,
+               "cannot use ScandirIterator from multiple threads concurrently"))
+        self._in_next = True
+        try:
+            #
+            space = self.space
+            while True:
+                try:
+                    entry = rposix_scandir.nextentry(self.dirp)
+                except OSError as e:
+                    self.fail(wrap_oserror(space, e))
+                if not entry:
+                    self.fail()
+                assert rposix_scandir.has_name_bytes(entry)
+                name = rposix_scandir.get_name_bytes(entry)
+                if name != '.' and name != '..':
+                    break
+            #
+            known_type = rposix_scandir.get_known_type(entry)
+        finally:
+            self._in_next = False
+        direntry = W_DirEntry(self, name, known_type)
         return space.wrap(direntry)
 
 
@@ -100,13 +106,31 @@
 W_ScandirIterator.typedef.acceptable_as_base_class = False
 
 
+class FileNotFound(Exception):
+    pass
+
+assert 0 <= rposix_scandir.DT_UNKNOWN <= 255
+assert 0 <= rposix_scandir.DT_REG <= 255
+assert 0 <= rposix_scandir.DT_DIR <= 255
+assert 0 <= rposix_scandir.DT_LNK <= 255
+FLAG_STAT  = 256
+FLAG_LSTAT = 512
+
+
 class W_DirEntry(W_Root):
     w_path = None
 
-    def __init__(self, w_name, w_path_prefix, result_is_bytes):
+    def __init__(self, scandir_iterator, name, known_type):
+        self.space = scandir_iterator.space
+        self.scandir_iterator = scandir_iterator
+        self.name = name     # always bytes on Posix
+        self.flags = known_type
+        assert known_type == (known_type & 255)
+        #
+        w_name = self.space.newbytes(name)
+        if not scandir_iterator.result_is_bytes:
+            w_name = self.space.fsdecode(w_name)
         self.w_name = w_name
-        self.w_path_prefix = w_path_prefix
-        self.result_is_bytes = result_is_bytes
 
     def fget_name(self, space):
         return self.w_name
@@ -114,10 +138,148 @@
     def fget_path(self, space):
         w_path = self.w_path
         if w_path is None:
-            w_path = space.add(self.w_path_prefix, self.w_name)
+            w_path_prefix = self.scandir_iterator.w_path_prefix
+            w_path = space.add(w_path_prefix, self.w_name)
             self.w_path = w_path
         return w_path
 
+    # The internal methods, used to implement the public methods at
+    # the end of the class.  Every method only calls methods *before*
+    # it in program order, so there is no cycle.
+
+    def get_lstat(self):
+        """Get the lstat() of the direntry."""
+        if (self.flags & FLAG_LSTAT) == 0:
+            # Unlike CPython, try to use fstatat() if possible
+            dirfd = self.scandir_iterator.dirfd
+            if dirfd != -1:
+                st = rposix_stat.fstatat(self.name, dirfd,
+                                         follow_symlinks=False)
+            else:
+                path = self.space.fsencode_w(self.fget_path(self.space))
+                st = rposix_stat.lstat(path)
+            self.d_lstat = st
+            self.flags |= FLAG_LSTAT
+        return self.d_lstat
+
+    def get_stat(self):
+        """Get the stat() of the direntry.  This is implemented in
+        such a way that it won't do both a stat() and a lstat().
+        """
+        if (self.flags & FLAG_STAT) == 0:
+            # We don't have the 'd_stat'.  If the known_type says the
+            # direntry is not a DT_LNK, then try to get and cache the
+            # 'd_lstat' instead.  Then, or if we already have a
+            # 'd_lstat' from before, *and* if the 'd_lstat' is not a
+            # S_ISLNK, we can reuse it unchanged for 'd_stat'.
+            #
+            # Note how, in the common case where the known_type says
+            # it is a DT_REG or DT_DIR, then we call and cache lstat()
+            # and that's it.  Also note that in a d_type-less OS or on
+            # a filesystem that always answer DT_UNKNOWN, this method
+            # will instead only call at most stat(), but not cache it
+            # as 'd_lstat'.
+            known_type = self.flags & 255
+            if (known_type != rposix_scandir.DT_UNKNOWN and
+                known_type != rposix_scandir.DT_LNK):
+                self.get_lstat()    # fill the 'd_lstat' cache
+                have_lstat = True
+            else:
+                have_lstat = (self.flags & FLAG_LSTAT) != 0
+
+            if have_lstat:
+                # We have the lstat() but not the stat().  They are
+                # the same, unless the 'd_lstat' is a S_IFLNK.
+                must_call_stat = stat.S_ISLNK(self.d_lstat.st_mode)
+            else:
+                must_call_stat = True
+
+            if must_call_stat:
+                # Must call stat().  Try to use fstatat() if possible
+                dirfd = self.scandir_iterator.dirfd
+                if dirfd != -1:
+                    st = rposix_stat.fstatat(self.name, dirfd,
+                                             follow_symlinks=True)
+                else:
+                    path = self.space.fsencode_w(self.fget_path(self.space))
+                    st = rposix_stat.stat(path)
+            else:
+                st = self.d_lstat
+
+            self.d_stat = st
+            self.flags |= FLAG_STAT
+        return self.d_stat
+
+    def get_stat_or_lstat(self, follow_symlinks):
+        if follow_symlinks:
+            return self.get_stat()
+        else:
+            return self.get_lstat()
+
+    def check_mode(self, follow_symlinks):
+        """Get the stat() or lstat() of the direntry, and return the
+        S_IFMT.  If calling stat()/lstat() gives us ENOENT, return -1
+        instead; it is better to give up and answer "no, not this type"
+        to requests, rather than propagate the error.
+        """
+        try:
+            st = self.get_stat_or_lstat(follow_symlinks)
+        except OSError as e:
+            if e.errno == ENOENT:    # not found
+                return -1
+            raise wrap_oserror2(self.space, e, self.fget_path(self.space))
+        return stat.S_IFMT(st.st_mode)
+
+    def is_dir(self, follow_symlinks):
+        known_type = self.flags & 255
+        if known_type != rposix_scandir.DT_UNKNOWN:
+            if known_type == rposix_scandir.DT_DIR:
+                return True
+            elif follow_symlinks and known_type == rposix_scandir.DT_LNK:
+                pass    # don't know in this case
+            else:
+                return False
+        return self.check_mode(follow_symlinks) == stat.S_IFDIR
+
+    def is_file(self, follow_symlinks):
+        known_type = self.flags & 255
+        if known_type != rposix_scandir.DT_UNKNOWN:
+            if known_type == rposix_scandir.DT_REG:
+                return True
+            elif follow_symlinks and known_type == rposix_scandir.DT_LNK:
+                pass    # don't know in this case
+            else:
+                return False
+        return self.check_mode(follow_symlinks) == stat.S_IFREG
+
+    def is_symlink(self):
+        """Check if the direntry is a symlink.  May get the lstat()."""
+        known_type = self.flags & 255
+        if known_type != rposix_scandir.DT_UNKNOWN:
+            return known_type == rposix_scandir.DT_LNK
+        return self.check_mode(follow_symlinks=False) == stat.S_IFLNK
+
+    @unwrap_spec(follow_symlinks=int)
+    def descr_is_dir(self, space, __kwonly__, follow_symlinks=1):
+        """return True if the entry is a directory; cached per entry"""
+        return space.wrap(self.is_dir(follow_symlinks))
+
+    @unwrap_spec(follow_symlinks=int)
+    def descr_is_file(self, space, __kwonly__, follow_symlinks=1):
+        """return True if the entry is a file; cached per entry"""
+        return space.wrap(self.is_file(follow_symlinks))
+
+    def descr_is_symlink(self, space):
+        """return True if the entry is a symbolic link; cached per entry"""
+        return space.wrap(self.is_symlink())
+
+    @unwrap_spec(follow_symlinks=int)
+    def descr_stat(self, space, __kwonly__, follow_symlinks=1):
+        """return stat_result object for the entry; cached per entry"""
+        st = self.get_stat_or_lstat(follow_symlinks)
+        return build_stat_result(self.space, st)
+
+
 W_DirEntry.typedef = TypeDef(
     'posix.DirEntry',
     name = GetSetProperty(W_DirEntry.fget_name,
@@ -126,5 +288,9 @@
     path = GetSetProperty(W_DirEntry.fget_path,
                           doc="the entry's full path name; equivalent to "
                               "os.path.join(scandir_path, entry.name)"),
+    is_dir = interp2app(W_DirEntry.descr_is_dir),
+    is_file = interp2app(W_DirEntry.descr_is_file),
+    is_symlink = interp2app(W_DirEntry.descr_is_symlink),
+    stat = interp2app(W_DirEntry.descr_stat),
 )
 W_DirEntry.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -7,7 +7,22 @@
     d = os.path.join(str(udir), dirname)
     os.mkdir(d)
     for key, value in content.items():
-        xxx
+        filename = os.path.join(d, key)
+        if value == 'dir':
+            os.mkdir(filename)
+        elif value == 'file':
+            with open(filename, 'w') as f:
+                pass
+        elif value == 'symlink-file':
+            os.symlink(str(udir.ensure('some_file')), filename)
+        elif value == 'symlink-dir':
+            os.symlink(str(udir), filename)
+        elif value == 'symlink-broken':
+            os.symlink(filename + '-broken', filename)
+        elif value == 'symlink-error':
+            os.symlink(filename, filename)
+        else:
+            raise NotImplementedError(repr(value))
     return d.decode(sys.getfilesystemencoding())
 
 
@@ -18,6 +33,15 @@
         space = cls.space
         cls.w_posix = space.appexec([], test_posix2.GET_POSIX)
         cls.w_dir_empty = space.wrap(_make_dir('empty', {}))
+        cls.w_dir0 = space.wrap(_make_dir('dir0', {'f1': 'file',
+                                                   'f2': 'file',
+                                                   'f3': 'file'}))
+        cls.w_dir1 = space.wrap(_make_dir('dir1', {'file1': 'file'}))
+        cls.w_dir2 = space.wrap(_make_dir('dir2', {'subdir2': 'dir'}))
+        cls.w_dir3 = space.wrap(_make_dir('dir3', {'sfile3': 'symlink-file'}))
+        cls.w_dir4 = space.wrap(_make_dir('dir4', {'sdir4': 'symlink-dir'}))
+        cls.w_dir5 = space.wrap(_make_dir('dir5', {'sbrok5': 'symlink-broken'}))
+        cls.w_dir6 = space.wrap(_make_dir('dir6', {'serr6': 'symlink-error'}))
 
     def test_scandir_empty(self):
         posix = self.posix
@@ -25,12 +49,22 @@
         assert list(sd) == []
         assert list(sd) == []
 
+    def test_scandir_files(self):
+        posix = self.posix
+        sd = posix.scandir(self.dir0)
+        names = [d.name for d in sd]
+        assert sorted(names) == ['f1', 'f2', 'f3']
+
     def test_unicode_versus_bytes(self):
         posix = self.posix
         d = next(posix.scandir())
         assert type(d.name) is str
         assert type(d.path) is str
         assert d.path == './' + d.name
+        d = next(posix.scandir(None))
+        assert type(d.name) is str
+        assert type(d.path) is str
+        assert d.path == './' + d.name
         d = next(posix.scandir(u'.'))
         assert type(d.name) is str
         assert type(d.path) is str
@@ -47,3 +81,79 @@
         assert type(d.name) is bytes
         assert type(d.path) is bytes
         assert d.path == b'/' + d.name
+
+    def test_stat1(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert d.name == 'file1'
+        assert d.stat().st_mode & 0o170000 == 0o100000    # S_IFREG
+        assert d.stat().st_size == 0
+
+    def test_stat4(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir4))
+        assert d.name == 'sdir4'
+        assert d.stat().st_mode & 0o170000 == 0o040000    # S_IFDIR
+        assert d.stat(follow_symlinks=True).st_mode &0o170000 == 0o040000
+        assert d.stat(follow_symlinks=False).st_mode&0o170000 == 0o120000 #IFLNK
+
+    def test_dir1(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert d.name == 'file1'
+        assert     d.is_file()
+        assert not d.is_dir()
+        assert not d.is_symlink()
+        raises(TypeError, d.is_file, True)
+        assert     d.is_file(follow_symlinks=False)
+        assert not d.is_dir(follow_symlinks=False)
+
+    def test_dir2(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir2))
+        assert d.name == 'subdir2'
+        assert not d.is_file()
+        assert     d.is_dir()
+        assert not d.is_symlink()
+        assert not d.is_file(follow_symlinks=False)
+        assert     d.is_dir(follow_symlinks=False)
+
+    def test_dir3(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir3))
+        assert d.name == 'sfile3'
+        assert     d.is_file()
+        assert not d.is_dir()
+        assert     d.is_symlink()
+        assert     d.is_file(follow_symlinks=True)
+        assert not d.is_file(follow_symlinks=False)
+
+    def test_dir4(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir4))
+        assert d.name == 'sdir4'
+        assert not d.is_file()
+        assert     d.is_dir()
+        assert     d.is_symlink()
+        assert     d.is_dir(follow_symlinks=True)
+        assert not d.is_dir(follow_symlinks=False)
+
+    def test_dir5(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir5))
+        assert d.name == 'sbrok5'
+        assert not d.is_file()
+        assert not d.is_dir()
+        assert     d.is_symlink()
+
+    def test_dir6(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir6))
+        assert d.name == 'serr6'
+        raises(OSError, d.is_file)
+        raises(OSError, d.is_dir)
+        assert d.is_symlink()
+
+    def test_fdopendir_unsupported(self):
+        posix = self.posix
+        raises(TypeError, posix.scandir, 1234)

From pypy.commits at gmail.com  Mon Aug 22 05:03:31 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 22 Aug 2016 02:03:31 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: can only return a W_Root
 object to app level (translation issue)
Message-ID: <57babfe3.94071c0a.b851.1b94@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86403:5599a214c3e4
Date: 2016-08-22 11:02 +0200
http://bitbucket.org/pypy/pypy/changeset/5599a214c3e4/

Log:	can only return a W_Root object to app level (translation issue)

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -19,6 +19,7 @@
     decode_object, unicode_from_encoded_object,
     unicode_from_string, getdefaultencoding)
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
+from pypy.objspace.std.strbuf import W_StringBufferObject
 
 
 class W_AbstractBytesObject(W_Root):
@@ -434,7 +435,6 @@
         of the specified width. The string S is never truncated.
         """
 
-
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
@@ -465,7 +465,7 @@
                     "Cannot use string as modifiable buffer")
 
     def descr_getbuffer(self, space, w_flags):
-        return StringBuffer(self._value)
+        return W_StringBufferObject(StringBuffer(self._value))
 
     charbuf_w = str_w
 

From pypy.commits at gmail.com  Mon Aug 22 05:08:03 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 22 Aug 2016 02:08:03 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: typo,
 module name incomplete, moved import to avoid cyclic dep.
Message-ID: <57bac0f3.262ec20a.11a57.fb00@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86404:5f1682828946
Date: 2016-08-22 11:07 +0200
http://bitbucket.org/pypy/pypy/changeset/5f1682828946/

Log:	typo, module name incomplete, moved import to avoid cyclic dep.

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -19,7 +19,6 @@
     decode_object, unicode_from_encoded_object,
     unicode_from_string, getdefaultencoding)
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
-from pypy.objspace.std.strbuf import W_StringBufferObject
 
 
 class W_AbstractBytesObject(W_Root):
@@ -465,6 +464,7 @@
                     "Cannot use string as modifiable buffer")
 
     def descr_getbuffer(self, space, w_flags):
+        from pypy.objspace.std.strbufobject import W_StringBufferObject
         return W_StringBufferObject(StringBuffer(self._value))
 
     charbuf_w = str_w

From pypy.commits at gmail.com  Mon Aug 22 05:21:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 02:21:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <57bac418.448e1c0a.6ef0d.10c7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86405:49cf5d090872
Date: 2016-08-22 11:20 +0200
http://bitbucket.org/pypy/pypy/changeset/49cf5d090872/

Log:	fix test

diff --git a/pypy/module/_sre/test/support_test_app_sre.py b/pypy/module/_sre/test/support_test_app_sre.py
--- a/pypy/module/_sre/test/support_test_app_sre.py
+++ b/pypy/module/_sre/test/support_test_app_sre.py
@@ -1,6 +1,13 @@
 """Support functions for app-level _sre tests."""
 import locale, _sre
-from sre_constants import OPCODES, ATCODES, CHCODES, MAXREPEAT
+from sre_constants import OPCODES as _OPCODES
+from sre_constants import ATCODES as _ATCODES
+from sre_constants import CHCODES as _CHCODES
+from sre_constants import MAXREPEAT
+
+OPCODES = {_opcode.name.lower(): int(_opcode) for _opcode in _OPCODES}
+ATCODES = {_atcode.name.lower(): int(_atcode) for _atcode in _ATCODES}
+CHCODES = {_chcode.name.lower(): int(_chcode) for _chcode in _CHCODES}
 
 def encode_literal(string):
     opcodes = []

From pypy.commits at gmail.com  Mon Aug 22 05:30:42 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 22 Aug 2016 02:30:42 -0700 (PDT)
Subject: [pypy-commit] pypy const-fold-we-are-jitted: document branch
Message-ID: <57bac642.11051c0a.d4d95.21f3@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: const-fold-we-are-jitted
Changeset: r86406:f49777711669
Date: 2016-08-22 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/f49777711669/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -144,3 +144,9 @@
 ``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
 Previously it returned what looked like a regular dict object (but it
 was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.

From pypy.commits at gmail.com  Mon Aug 22 05:32:57 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 22 Aug 2016 02:32:57 -0700 (PDT)
Subject: [pypy-commit] pypy const-fold-we-are-jitted: close to-be-merged
 branch
Message-ID: <57bac6c9.45c8c20a.762ba.11e7@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: const-fold-we-are-jitted
Changeset: r86407:69dfdb6dd204
Date: 2016-08-22 11:30 +0200
http://bitbucket.org/pypy/pypy/changeset/69dfdb6dd204/

Log:	close to-be-merged branch


From pypy.commits at gmail.com  Mon Aug 22 05:32:59 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 22 Aug 2016 02:32:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge const-fold-we-are-jitted:
Message-ID: <57bac6cb.6211c20a.3ee9e.0761@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86408:532f12cdb886
Date: 2016-08-22 11:32 +0200
http://bitbucket.org/pypy/pypy/changeset/532f12cdb886/

Log:	merge const-fold-we-are-jitted:

	when running the final backend-optimization phase before emitting C
	code, constant-fold calls to we_are_jitted to return False. This
	makes the generated C code a few percent smaller.

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -144,3 +144,9 @@
 ``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
 Previously it returned what looked like a regular dict object (but it
 was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -261,6 +261,9 @@
                    "stack based virtual machines (only for backends that support it)",
                    default=True),
         BoolOption("storesink", "Perform store sinking", default=True),
+        BoolOption("replace_we_are_jitted",
+                   "Replace we_are_jitted() calls by False",
+                   default=False, cmdline=None),
         BoolOption("none",
                    "Do not run any backend optimizations",
                    requires=[('translation.backendopt.inline', False),
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -452,7 +452,8 @@
                               merge_if_blocks=True,
                               constfold=True,
                               remove_asserts=True,
-                              really_remove_asserts=True)
+                              really_remove_asserts=True,
+                              replace_we_are_jitted=False)
 
     def prejit_optimizations_minimal_inline(self, policy, graphs):
         from rpython.translator.backendopt.inline import auto_inline_graphs
diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py
--- a/rpython/translator/backendopt/all.py
+++ b/rpython/translator/backendopt/all.py
@@ -2,6 +2,7 @@
 from rpython.translator.backendopt import inline
 from rpython.translator.backendopt.malloc import remove_mallocs
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.translator.backendopt.stat import print_statistics
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator import simplify
@@ -36,6 +37,7 @@
     # inline_threshold, mallocs
     # merge_if_blocks, constfold, heap2stack
     # clever_malloc_removal, remove_asserts
+    # replace_we_are_jitted
 
     config = translator.config.translation.backendopt.copy(as_default=True)
     config.set(**kwds)
@@ -49,6 +51,10 @@
         print "before optimizations:"
         print_statistics(translator.graphs[0], translator, "per-graph.txt")
 
+    if config.replace_we_are_jitted:
+        for graph in graphs:
+            replace_we_are_jitted(graph)
+
     if config.remove_asserts:
         constfold(config, graphs)
         remove_asserts(translator, graphs)
diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py
--- a/rpython/translator/backendopt/constfold.py
+++ b/rpython/translator/backendopt/constfold.py
@@ -276,3 +276,25 @@
             rewire_links(splitblocks, graph)
         if not diffused and not splitblocks:
             break # finished
+
+def replace_symbolic(graph, symbolic, value):
+    result = False
+    for block in graph.iterblocks():
+        for op in block.operations:
+            for i, arg in enumerate(op.args):
+                if isinstance(arg, Constant) and arg.value is symbolic:
+                    op.args[i] = value
+                    result = True
+        if block.exitswitch is symbolic:
+            block.exitswitch = value
+            result = True
+    return result
+
+def replace_we_are_jitted(graph):
+    from rpython.rlib import jit
+    replacement = Constant(0)
+    replacement.concretetype = lltype.Signed
+    did_replacement = replace_symbolic(graph, jit._we_are_jitted, replacement)
+    if did_replacement:
+        constant_fold_graph(graph)
+    return did_replacement
diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py
--- a/rpython/translator/backendopt/test/test_all.py
+++ b/rpython/translator/backendopt/test/test_all.py
@@ -289,3 +289,19 @@
         llinterp = LLInterpreter(t.rtyper)
         res = llinterp.eval_graph(later_graph, [10])
         assert res == 1
+
+    def test_replace_we_are_jitted(self):
+        from rpython.rlib import jit
+        def f():
+            if jit.we_are_jitted():
+                return 1
+            return 2 + jit.we_are_jitted()
+
+        t = self.translateopt(f, [])
+        graph = graphof(t, f)
+        # by default, replace_we_are_jitted is off
+        assert graph.startblock.operations[0].args[0].value is jit._we_are_jitted
+
+        t = self.translateopt(f, [], replace_we_are_jitted=True)
+        graph = graphof(t, f)
+        assert graph.startblock.exits[0].args[0].value == 2
diff --git a/rpython/translator/backendopt/test/test_constfold.py b/rpython/translator/backendopt/test/test_constfold.py
--- a/rpython/translator/backendopt/test/test_constfold.py
+++ b/rpython/translator/backendopt/test/test_constfold.py
@@ -7,6 +7,7 @@
 from rpython.rtyper import rclass
 from rpython.rlib import objectmodel
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.conftest import option
 
 def get_graph(fn, signature):
@@ -343,3 +344,18 @@
     merge_if_blocks.merge_if_blocks_once(graph)
     constant_fold_graph(graph)
     check_graph(graph, [], 66, t)
+
+def test_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def fn():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+    graph, t = get_graph(fn, [])
+    result = replace_we_are_jitted(graph)
+    assert result
+    checkgraph(graph)
+    # check shape of graph
+    assert len(graph.startblock.operations) == 0
+    assert graph.startblock.exitswitch is None
+    assert graph.startblock.exits[0].target.exits[0].args[0].value == 2
diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -381,7 +381,7 @@
         """ Run all backend optimizations - lltype version
         """
         from rpython.translator.backendopt.all import backend_optimizations
-        backend_optimizations(self.translator)
+        backend_optimizations(self.translator, replace_we_are_jitted=True)
 
 
     STACKCHECKINSERTION = 'stackcheckinsertion_lltype'
diff --git a/rpython/translator/test/test_interactive.py b/rpython/translator/test/test_interactive.py
--- a/rpython/translator/test/test_interactive.py
+++ b/rpython/translator/test/test_interactive.py
@@ -78,3 +78,15 @@
     dll = ctypes.CDLL(str(t.driver.c_entryp))
     f = dll.pypy_g_f
     assert f(2, 3) == 5
+
+def test_check_that_driver_uses_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def f():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+
+    t = Translation(f, [])
+    t.backendopt()
+    graph = t.driver.translator.graphs[0]
+    assert graph.startblock.exits[0].args[0].value == 2

From pypy.commits at gmail.com  Mon Aug 22 05:38:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 22 Aug 2016 02:38:59 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: add newbytes method to
 tinyobjspace (as cfbolz suggested)
Message-ID: <57bac833.43681c0a.d0d8.25a3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86409:092eb3a20315
Date: 2016-08-22 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/092eb3a20315/

Log:	add newbytes method to tinyobjspace (as cfbolz suggested)

diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -10,10 +10,10 @@
 
     def setup_class(cls):
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__jitlog.1')))
-        cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
-        cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
+        cls.w_mark_header = cls.space.newbytes(jl.MARK_JITLOG_HEADER)
+        cls.w_version = cls.space.newbytes(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
-        cls.w_machine = cls.space.wrap(platform.machine())
+        cls.w_machine = cls.space.newbytes(platform.machine())
         cls.w_resops = cls.space.newdict()
         space = cls.space
         for key, value in opname.items():
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -107,6 +107,9 @@
     def newlist(self, iterable):
         return list(iterable)
 
+    def newbytes(self, obj):
+        return bytes(obj)
+
     def call_function(self, func, *args, **kwds):
         return func(*args, **kwds)
 

From pypy.commits at gmail.com  Mon Aug 22 05:47:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 02:47:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: typo
Message-ID: <57baca1d.e2efc20a.ba596.04a0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86410:85dea6f34028
Date: 2016-08-22 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/85dea6f34028/

Log:	typo

diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -799,8 +799,8 @@
                            rffi.cast(size_t, map_size),
                            rffi.cast(rffi.INT, use_flag))
     else:
-        def madvice_free(addr, map_size):
-            "No madvice() on this platform"
+        def madvise_free(addr, map_size):
+            "No madvise() on this platform"
 
 elif _MS_WINDOWS:
     def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):

From pypy.commits at gmail.com  Mon Aug 22 05:47:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 02:47:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix for posix/test/test_ztranslation
Message-ID: <57baca37.53b81c0a.915fe.2580@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86411:cd1610618bd6
Date: 2016-08-22 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/cd1610618bd6/

Log:	Fix for posix/test/test_ztranslation

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -430,6 +430,8 @@
     def get(self, name):
         name + "xx"   # check that it's a string
         return w_some_obj()
+    def setmodule(self, w_mod):
+        is_root(w_mod)
 FakeObjSpace.sys = FakeModule()
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'

From pypy.commits at gmail.com  Mon Aug 22 05:47:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 02:47:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: translation fix
Message-ID: <57baca38.898b1c0a.1c93f.352b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86412:0b690c524507
Date: 2016-08-22 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/0b690c524507/

Log:	translation fix

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -71,9 +71,9 @@
 
     def next_w(self):
         if not self.dirp:
-            self.fail()
+            raise self.fail()
         if self._in_next:
-            self.fail(oefmt(self.space.w_RuntimeError,
+            raise self.fail(oefmt(self.space.w_RuntimeError,
                "cannot use ScandirIterator from multiple threads concurrently"))
         self._in_next = True
         try:
@@ -83,9 +83,9 @@
                 try:
                     entry = rposix_scandir.nextentry(self.dirp)
                 except OSError as e:
-                    self.fail(wrap_oserror(space, e))
+                    raise self.fail(wrap_oserror2(space, e, self.w_path_prefix))
                 if not entry:
-                    self.fail()
+                    raise self.fail()
                 assert rposix_scandir.has_name_bytes(entry)
                 name = rposix_scandir.get_name_bytes(entry)
                 if name != '.' and name != '..':
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -17,7 +17,7 @@
 def closedir(dirp):
     rposix.c_closedir(dirp)
 
-NULL_DIRP = lltype.nullptr(rposix.DIRENT)
+NULL_DIRP = lltype.nullptr(rposix.DIRP.TO)
 
 def nextentry(dirp):
     """Read the next entry and returns an opaque object.
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -648,6 +648,7 @@
 
     @staticmethod
     @jit.elidable
+    @signature(types.any(), types.any(), types.int(), types.int(), returns=types.int())
     def ll_rfind_char(s, ch, start, end):
         if end > len(s.chars):
             end = len(s.chars)

From pypy.commits at gmail.com  Mon Aug 22 05:48:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 02:48:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: backport 0b690c524507
Message-ID: <57baca53.8aacc20a.9e40d.118e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86413:b78a73998610
Date: 2016-08-22 11:47 +0200
http://bitbucket.org/pypy/pypy/changeset/b78a73998610/

Log:	backport 0b690c524507

diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -17,7 +17,7 @@
 def closedir(dirp):
     rposix.c_closedir(dirp)
 
-NULL_DIRP = lltype.nullptr(rposix.DIRENT)
+NULL_DIRP = lltype.nullptr(rposix.DIRP.TO)
 
 def nextentry(dirp):
     """Read the next entry and returns an opaque object.
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -648,6 +648,7 @@
 
     @staticmethod
     @jit.elidable
+    @signature(types.any(), types.any(), types.int(), types.int(), returns=types.int())
     def ll_rfind_char(s, ch, start, end):
         if end > len(s.chars):
             end = len(s.chars)

From pypy.commits at gmail.com  Mon Aug 22 06:07:03 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 22 Aug 2016 03:07:03 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: renamed
 W_StringBufferObject to W_StringBuilderObject (really a string builder is
 passed), the new object W_StringBufferObject now takes and wrapps a
 rlib.buffer:Buffer object!
Message-ID: <57bacec7.ca11c30a.e4f0f.1b27@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86414:4cd49fa0d0d9
Date: 2016-08-22 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/4cd49fa0d0d9/

Log:	renamed W_StringBufferObject to W_StringBuilderObject (really a
	string builder is passed), the new object W_StringBufferObject now
	takes and wrapps a rlib.buffer:Buffer object!

diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -431,7 +431,7 @@
 :source:`pypy/objspace/std/bytesobject.py` defines ``W_AbstractBytesObject``,
 which contains everything needed to build the ``str`` app-level type;
 and there are subclasses ``W_BytesObject`` (the usual string) and
-``W_StringBufferObject`` (a special implementation tweaked for repeated
+``W_StringBuilderObject`` (a special implementation tweaked for repeated
 additions, in :source:`pypy/objspace/std/strbufobject.py`).  For mutable data
 types like lists and dictionaries, we have a single class
 ``W_ListObject`` or ``W_DictMultiObject`` which has an indirection to
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -603,8 +603,8 @@
 
     def descr_eq(self, space, w_other):
         if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
+            from pypy.objspace.std.strbufobject import W_StringBuilderObject
+            if isinstance(w_other, W_StringBuilderObject):
                 return space.newbool(self._value == w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
@@ -612,8 +612,8 @@
 
     def descr_ne(self, space, w_other):
         if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
+            from pypy.objspace.std.strbufobject import W_StringBuilderObject
+            if isinstance(w_other, W_StringBuilderObject):
                 return space.newbool(self._value != w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
@@ -621,8 +621,8 @@
 
     def descr_lt(self, space, w_other):
         if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
+            from pypy.objspace.std.strbufobject import W_StringBuilderObject
+            if isinstance(w_other, W_StringBuilderObject):
                 return space.newbool(self._value < w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
@@ -630,8 +630,8 @@
 
     def descr_le(self, space, w_other):
         if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
+            from pypy.objspace.std.strbufobject import W_StringBuilderObject
+            if isinstance(w_other, W_StringBuilderObject):
                 return space.newbool(self._value <= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
@@ -639,8 +639,8 @@
 
     def descr_gt(self, space, w_other):
         if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
+            from pypy.objspace.std.strbufobject import W_StringBuilderObject
+            if isinstance(w_other, W_StringBuilderObject):
                 return space.newbool(self._value > w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
@@ -648,8 +648,8 @@
 
     def descr_ge(self, space, w_other):
         if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
+            from pypy.objspace.std.strbufobject import W_StringBuilderObject
+            if isinstance(w_other, W_StringBuilderObject):
                 return space.newbool(self._value >= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
@@ -669,7 +669,7 @@
             self_as_bytearray = W_BytearrayObject(_make_data(self._value))
             return space.add(self_as_bytearray, w_other)
         if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
+            from pypy.objspace.std.strbufobject import W_StringBuilderObject
             try:
                 other = self._op_val(space, w_other)
             except OperationError as e:
@@ -679,7 +679,7 @@
             builder = StringBuilder()
             builder.append(self._value)
             builder.append(other)
-            return W_StringBufferObject(builder)
+            return W_StringBuilderObject(builder)
         return self._StringMethods_descr_add(space, w_other)
 
     _StringMethods__startswith = _startswith
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -8,8 +8,34 @@
 from pypy.interpreter.error import OperationError
 from rpython.rlib.rstring import StringBuilder
 
+class W_StringBufferObject(W_AbstractBytesObject):
+    # this wraps an rpython/rlib/buffer.py:Buffer object, not a StringBuilder
+    # NOTE the W_StringBuilderObject *was* named W_StringBufferObject
+    w_str = None
 
-class W_StringBufferObject(W_AbstractBytesObject):
+    def __init__(self, buffer):
+        self.buffer = buffer # Buffer
+        self.length = buffer.getlength()
+
+    def __repr__(self):
+        """ representation for debugging purposes """
+        return "%s(%r[:%d])" % (
+            self.__class__.__name__, self.buffer, self.length)
+
+    def unwrap(self, space):
+        return self.buffer.value
+
+    def str_w(self, space):
+        return self.buffer.value
+
+    def buffer_w(self, space, flags):
+        return self.buffer
+
+    def readbuf_w(self, space):
+        return self.buffer
+
+
+class W_StringBuilderObject(W_AbstractBytesObject):
     w_str = None
 
     def __init__(self, builder):
@@ -59,11 +85,11 @@
         else:
             builder = self.builder
         builder.append(other)
-        return W_StringBufferObject(builder)
+        return W_StringBuilderObject(builder)
 
     def descr_str(self, space):
-        # you cannot get subclasses of W_StringBufferObject here
-        assert type(self) is W_StringBufferObject
+        # you cannot get subclasses of W_StringBuilderObject here
+        assert type(self) is W_StringBuilderObject
         return self
 
 
@@ -94,6 +120,6 @@
     unwrap_spec_ = getattr(func, 'unwrap_spec', None)
     if unwrap_spec_ is not None:
         f = unwrap_spec(**unwrap_spec_)(f)
-    setattr(W_StringBufferObject, func.func_name, f)
+    setattr(W_StringBuilderObject, func.func_name, f)
 
-W_StringBufferObject.typedef = W_BytesObject.typedef
+W_StringBuilderObject.typedef = W_BytesObject.typedef
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ b/pypy/objspace/std/test/test_strbufobject.py
@@ -11,7 +11,7 @@
         # away on AST level
         s = "Hello, ".__add__("World!")
         assert type(s) is str
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
+        assert 'W_StringBuilderObject' in __pypy__.internal_repr(s)
 
     def test_add_twice(self):
         x = "a".__add__("b")
@@ -25,7 +25,7 @@
         all = ""
         for i in range(20):
             all += str(i)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(all)
+        assert 'W_StringBuilderObject' in __pypy__.internal_repr(all)
         assert all == "012345678910111213141516171819"
 
     def test_hash(self):
@@ -33,7 +33,7 @@
         def join(s): return s[:len(s) // 2] + s[len(s) // 2:]
         t = 'a' * 101
         s = join(t)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
+        assert 'W_StringBuilderObject' in __pypy__.internal_repr(s)
         assert hash(s) == hash(t)
 
     def test_len(self):

From pypy.commits at gmail.com  Mon Aug 22 07:18:48 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 22 Aug 2016 04:18:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: lots of logic for memoryview
 (work in progress)
Message-ID: <57badf98.81a2c20a.9de76.3a9d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86415:970730c55763
Date: 2016-08-22 13:18 +0200
http://bitbucket.org/pypy/pypy/changeset/970730c55763/

Log:	lots of logic for memoryview (work in progress)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1428,7 +1428,6 @@
 
     BUF_SIMPLE   = 0x0000
     BUF_WRITABLE = 0x0001
-    BUF_C        = 0x0002
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
@@ -1440,6 +1439,12 @@
     BUF_FULL_RO = BUF_INDIRECT | BUF_FORMAT
     BUF_FULL    = BUF_INDIRECT | BUF_FORMAT | BUF_WRITABLE
 
+    MEMORYVIEW_MAX_DIM = 64
+    MEMORYVIEW_C        = 0x0002
+    MEMORYVIEW_FORTRAN  = 0x0004
+    MEMORYVIEW_SCLAR    = 0x0008
+    MEMORYVIEW_PIL      = 0x0010
+
     def check_buf_flags(self, flags, readonly):
         if readonly and flags & self.BUF_WRITABLE == self.BUF_WRITABLE:
             raise oefmt(self.w_BufferError, "Object is not writable.")
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -225,6 +225,14 @@
             size = rffi.sizeof(rffi.VOIDP)
         return size
 
+    def _zero_in_shape(self):
+        view = self.buf
+        shape = view.shape
+        for i in range(view.ndim):
+            if shape[i] == 0:
+                return True
+        return False
+
     def descr_cast(self, space, w_format, w_shape=None):
         self._check_released(space)
 
@@ -232,21 +240,111 @@
             raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: format argument must be a string"))
 
-        # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
-        #if self.flags & (space.BUF_CONTIG_RO|space.BUF_C) == 0:
-        #    raise OperationError(space.w_TypeError, \
-        #            space.wrap("memoryview: casts are restricted" \
-        #                       " to C-contiguous views"))
+        buf = self.buf
+        ndim = 1
 
-        fmt = space.str_w(w_format)
-        newitemsize = self.get_native_fmtchar(fmt)
-        return W_MemoryView(self.buf, fmt, newitemsize)
+        if not memory_view_c_contiguous(buf.flags):
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: casts are restricted" \
+                               " to C-contiguous views"))
+
+        if (w_shape or buf.ndim != 1) and self._zero_in_shape():
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: cannot casts view with" \
+                               " zeros in shape or strides"))
+
+        if w_shape:
+            if not (space.is_w(w_obj, space.w_list) or space.is_w(w_obj, space.w_tuple)):
+                raise oefmt(space.w_TypeError, "expected list or tuple got %T", w_obj)
+            ndim = space.len_w(w_obj)
+            if ndim > space.BUF_MAX_DIM:
+                raise oefmt(space.w_ValueError, \
+                        "memoryview: number of dimensions must not exceed %d",
+                        ndim)
+            if ndim != buf.ndim:
+                raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: cast must be 1D -> ND or ND -> 1D"))
+
+        mv = W_MemoryView(self.buf)
+        mv._init_shared_values(space, self)
+        mv.itemsize = self.get_native_fmtchar(fmt)
+
+        if not mv._cast_to_1D(space, fmt):
+            return space.w_None
+        if w_shape is not space.w_None:
+            shape = space.fixedview(w_shape)
+            if not mv._cast_to_ND(space, shape, ndim):
+                return space.w_None
+        return mv
+
+    def _init_flags(self):
+        view = self.buf
+        ndim = view.ndim
+        flags = 0
+        if ndim == 0:
+            flags |= space.MEMORYVIEW_SCALAR | space.MEMORYVIEW_C | space.MEMORYVIEW_FORTRAN
+        if ndim == 1:
+            if view.shape[0] == 1 and view.strides[0] == view.itemsize:
+                flags |= space.MEMORYVIEW_C | space.MEMORYVIEW_SCALAR
+        if view.is_contiguous('C'):
+            flags |= space.MEMORYVIEW_C
+        elif view.is_contiguous('F'):
+            flags |= space.MEMORYVIEW_SCALAR
+
+        # XXX missing suboffsets
+
+        view.flags = flags
+
+    def _cast_to_1D(self, space, fmt):
+        itemsize = self.get_native_fmtchar(fmt)
+        view = self.buf
+        if itemsize < 0:
+            raise OperationError(space.w_ValueError, "memoryview: destination" \
+                    " format must be a native single character format prefixed" \
+                    " with an optional '@'")
+
+        if self.get_native_fmtchar(view.format) < 0 or \
+           (not is_byte_format(fmt) and not is_byte_format(view.format)):
+            raise OperationError(space.w_TypeError,
+                    "memoryview: cannot cast between" \
+                    " two non-byte formats")
+
+        if view.length % itemsize != 0:
+            raise OperationError(space.w_TypeError,
+                    "memoryview: length is not a multiple of itemsize")
+
+        view.format = get_native_fmtstr(fmt)
+        if not view.format:
+            raise OperationError(space.w_RuntimeError,
+                    "memoryview: internal error")
+        view.itemsize = itemsize
+        view.ndim = 1
+        view.shape[0] = view.length / view.itemsize
+        view.srides[0] = view.itemsize
+        # XX suboffsets
+
+        mv._init_flags()
+
+    def _cast_to_ND(self, space, shape, ndim):
+        pass
+
+    def _init_shared_values(self, space, of):
+        mv.buf = buf # XXX not quite right
+        mv.format = of.format
+        mv.readonly = of.readonly
+        mv.itemsize = of.itemsize
+        return mv
 
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         self._check_released(space)
         return _array_to_hexstring(space, self.buf)
 
+def is_byte_format(char):
+    return char == 'b' or char == 'B' or char == 'c'
+
+def memory_view_c_contiguous(flags):
+    return flags & (space.BUF_CONTIG_RO|space.BUF_C) != 0
 
 W_MemoryView.typedef = TypeDef(
     "memoryview",

From pypy.commits at gmail.com  Mon Aug 22 07:22:44 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 22 Aug 2016 04:22:44 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: space.newbool(...) rather
 than use space.wrap. tests require a bool is returned
Message-ID: <57bae084.03121c0a.636a9.601f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86416:ac02a70c592b
Date: 2016-08-22 13:22 +0200
http://bitbucket.org/pypy/pypy/changeset/ac02a70c592b/

Log:	space.newbool(...) rather than use space.wrap. tests require a bool
	is returned

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -127,7 +127,7 @@
         return space.wrap(self.buf.getndim())
 
     def w_is_readonly(self, space):
-        return space.wrap(self.buf.readonly)
+        return space.newbool(self.buf.readonly)
 
     def w_get_shape(self, space):
         return space.newtuple([space.wrap(x) for x in self.buf.getshape()])

From pypy.commits at gmail.com  Mon Aug 22 10:22:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 07:22:38 -0700 (PDT)
Subject: [pypy-commit] cffi default: Avoid calling more CPython functions
 when there is an exception set
Message-ID: <57bb0aae.a427c20a.70a77.856f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2739:12ee26189ebc
Date: 2016-08-22 15:10 +0200
http://bitbucket.org/cffi/cffi/changeset/12ee26189ebc/

Log:	Avoid calling more CPython functions when there is an exception set

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -4623,6 +4623,8 @@
                 ct = ct->ct_itemdescr;
             }
             ffifield = fb_fill_type(fb, ct, 0);
+            if (PyErr_Occurred())
+                return NULL;
             if (elements != NULL) {
                 for (j=0; j<flat; j++)
                     elements[nflat++] = ffifield;

From pypy.commits at gmail.com  Mon Aug 22 10:22:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 07:22:40 -0700 (PDT)
Subject: [pypy-commit] cffi default: Returning unions works fine in API mode,
 remove the checks.
Message-ID: <57bb0ab0.031dc20a.34dd5.8dce@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2740:34b29a139894
Date: 2016-08-22 15:26 +0200
http://bitbucket.org/cffi/cffi/changeset/34b29a139894/

Log:	Returning unions works fine in API mode, remove the checks.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3042,13 +3042,14 @@
 static PyObject *
 convert_struct_to_owning_object(char *data, CTypeDescrObject *ct)
 {
+    /* also accepts unions, for the API mode */
     CDataObject *cd;
     Py_ssize_t dataoffset = offsetof(CDataObject_own_nolength, alignment);
     Py_ssize_t datasize = ct->ct_size;
 
-    if ((ct->ct_flags & (CT_STRUCT|CT_IS_OPAQUE)) != CT_STRUCT) {
+    if (datasize < 0) {
         PyErr_SetString(PyExc_TypeError,
-                        "return type is not a struct or is opaque");
+                        "return type is an opaque structure or union");
         return NULL;
     }
     cd = allocate_owning_object(dataoffset + datasize, ct);
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))

From pypy.commits at gmail.com  Mon Aug 22 10:22:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 07:22:42 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update the docs according to issue #281
Message-ID: <57bb0ab2.a6a5c20a.d74e.8965@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2741:2bcd60790014
Date: 2016-08-22 15:54 +0200
http://bitbucket.org/cffi/cffi/changeset/2bcd60790014/

Log:	Update the docs according to issue #281

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -269,9 +269,7 @@
 
 Usually, the right thing to do is to call this method with True.  Be
 aware (particularly on Python 2) that, afterwards, you need to pass unicode
-strings as arguments instead of byte strings.  (Before cffi version 0.9,
-``TCHAR`` and friends where hard-coded as unicode, but ``UNICODE`` was,
-inconsistently, not defined by default.)
+strings as arguments instead of byte strings.
 
 
 .. _loading-libraries:
@@ -336,7 +334,7 @@
 
 **ffibuilder.set_source(module_name, c_header_source, [\*\*keywords...])**:
 prepare the ffi for producing out-of-line an external module called
-``module_name``.  *New in version 1.0.*
+``module_name``.
 
 ``ffibuilder.set_source()`` by itself does not write any file, but merely
 records its arguments for later.  It can therefore be called before or
@@ -425,7 +423,7 @@
    declaration which doesn't use "``...``" is assumed to be exact, but this is
    checked: you get an error if it is not correct.
 
-*  *New in version 1.1:* integer types: the syntax "``typedef
+*  integer types: the syntax "``typedef
    int... foo_t;``" declares the type ``foo_t`` as an integer type
    whose exact size and signedness is not specified.  The compiler will
    figure it out.  (Note that this requires ``set_source()``; it does
@@ -462,8 +460,8 @@
    length is completed by the C compiler.
    This is slightly different from "``int n[];``", because the latter
    means that the length is not known even to the C compiler, and thus
-   no attempt is made to complete it.  *New in version 1.1:* support
-   for multidimensional arrays: "``int n[...][...];``".
+   no attempt is made to complete it.  This supports
+   multidimensional arrays: "``int n[...][...];``".
 
    *New in version 1.2:* "``int m[][...];``", i.e. ``...`` can be used
    in the innermost dimensions without being also used in the outermost
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -292,7 +292,7 @@
 
 3. ``ffi.addressof(<library>, "name")`` returns the address of the
 named function or global variable from the given library object.
-*New in version 1.1:* for functions, it returns a regular cdata
+For functions, it returns a regular cdata
 object containing a pointer to the function.
 
 Note that the case 1. cannot be used to take the address of a
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -366,8 +366,8 @@
 
 __ ref.html#conversions
 
-CFFI supports passing and returning structs to functions and callbacks.
-Example:
+CFFI supports passing and returning structs and unions to functions and
+callbacks.  Example:
 
 .. code-block:: python
 
@@ -377,36 +377,33 @@
     myfoo = lib.function_returning_a_struct()
     # `myfoo`: <cdata 'struct foo_s' owning 8 bytes>
 
-There are a few (obscure) limitations to the argument types and return
-type.  You cannot pass directly as argument a union (but a *pointer*
-to a union is fine), nor a struct which uses bitfields (but a
-*pointer* to such a struct is fine).  If you pass a struct (not a
-*pointer* to a struct), the struct type cannot have been declared with
-"``...;``" in the ``cdef()``; you need to declare it completely in
-``cdef()``.  You can work around these limitations by writing a C
-function with a simpler signature in the C header code passed to
-``ffibuilder.set_source()``, and have this C function call the real one.
-
-Aside from these limitations, functions and callbacks can receive and
-return structs.
-
-For performance, API-level functions are not returned as ``<cdata>``
-objects, but as a different type (on CPython, ``<built-in
-function>``).  This means you cannot e.g. pass them to some other C
+For performance, non-variadic API-level functions that you get by
+writing ``lib.some_function`` are not ``<cdata>``
+objects, but an object of a different type (on CPython, ``<built-in
+function>``).  This means you cannot pass them directly to some other C
 function expecting a function pointer argument.  Only ``ffi.typeof()``
 works on them.  To get a cdata containing a regular function pointer,
-use ``ffi.addressof(lib, "name")`` (new in version 1.1).
+use ``ffi.addressof(lib, "name")``.
 
-Before version 1.1 (or with the deprecated ``ffi.verify()``), if you
-really need a cdata pointer to the function, use the following
-workaround:
+There are a few (obscure) limitations to the supported argument and
+return types.  These limitations come from libffi and apply only to
+calling ``<cdata>`` function pointers; in other words, they don't
+apply to non-variadic ``cdef()``-declared functions if you are using
+the API mode.  The limitations are that you cannot pass directly as
+argument or return type:
 
-.. code-block:: python
-  
-    ffi.cdef(""" int (*foo)(int a, int b); """)
+* a union (but a *pointer* to a union is fine);
 
-i.e. declare them as pointer-to-function in the cdef (even if they are
-regular functions in the C code).
+* a struct which uses bitfields (but a *pointer* to such a struct is
+  fine);
+
+* a struct that was declared with "``...``" in the ``cdef()``.
+
+In API mode, you can work around these limitations: for example, if you
+need to call such a function pointer from Python, you can instead write
+a custom C function that accepts the function pointer and the real
+arguments and that does the call from C.  Then declare that custom C
+function in the ``cdef()`` and use it from Python.
 
 
 Variadic function calls

From pypy.commits at gmail.com  Mon Aug 22 10:22:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 07:22:44 -0700 (PDT)
Subject: [pypy-commit] cffi default: Add two tests for 34b29a139894
Message-ID: <57bb0ab4.09afc20a.86a7e.87c7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2742:1362f9f5048a
Date: 2016-08-22 16:01 +0200
http://bitbucket.org/cffi/cffi/changeset/1362f9f5048a/

Log:	Add two tests for 34b29a139894

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1962,3 +1962,21 @@
                        ffi, "test_function_returns_opaque", "?")
     assert str(e.value) == ("function foo: 'struct a' is used as result type,"
                             " but is opaque")
+
+def test_function_returns_union():
+    ffi = FFI()
+    ffi.cdef("union u1 { int a, b; }; union u1 f1(int);")
+    lib = verify(ffi, "test_function_returns_union", """
+        union u1 { int a, b; };
+        static union u1 f1(int x) { union u1 u; u.b = x; return u; }
+    """)
+    assert lib.f1(51).a == 51
+
+def test_function_returns_partial_struct():
+    ffi = FFI()
+    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    lib = verify(ffi, "test_function_returns_partial_struct", """
+        struct a { int b, a, c; };
+        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+    """)
+    assert lib.f1(52).a == 52

From pypy.commits at gmail.com  Mon Aug 22 10:22:48 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 22 Aug 2016 07:22:48 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Stop trying to set tid via c code
 as it does not work
Message-ID: <57bb0ab8.04141c0a.e63b0.98ff@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86418:d4fd1b5588ee
Date: 2016-08-22 16:21 +0200
http://bitbucket.org/pypy/pypy/changeset/d4fd1b5588ee/

Log:	Stop trying to set tid via c code as it does not work

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -32,23 +32,21 @@
         ll_assert(not needs_finalizer, 'finalizer not supported')
         ll_assert(not is_finalizer_light, 'light finalizer not supported')
         ll_assert(not contains_weakptr, 'weakref not supported')
+        # FIXME: set typeid and hash here
         return llop.qcgc_allocate(llmemory.GCREF, size, typeid)
 
     def malloc_varsize_clear(self, typeid16, length, size, itemsize,
                              offset_to_length):
         totalsize = size + itemsize * length
-        totalsize = llarena.round_up_for_allocation(totalsize)
+        #totalsize = llarena.round_up_for_allocation(totalsize)
         obj = llop.qcgc_allocate(llmemory.Address, totalsize, typeid16)
         (obj + offset_to_length).signed[0] = length
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     def collect(self, gen=1):
         """Do a minor (gen=0) or major (gen>0) collection."""
-        raise NotImplementedError
-        #if gen > 0:
-        #    llop.stm_major_collect(lltype.Void)
-        #else:
-        #    llop.stm_minor_collect(lltype.Void)
+        # XXX: Minor collection not supported
+        llop.qcgc_collect(lltype.Void)
 
     def writebarrier_before_copy(self, source_addr, dest_addr,
                                  source_start, dest_start, length):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -516,6 +516,7 @@
 
     # __________ qcgc operations __________
     'qcgc_allocate':    LLOp(canmallocgc=True),
+    'qcgc_collect':     LLOp(), # XXX: No allocations, so no canmallocgc ?
 
     # __________ weakrefs __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -956,5 +956,7 @@
         size = self.expr(op.args[0])
         typeid = self.expr(op.args[1])
         result = self.expr(op.result)
-        return ('%s = qcgc_allocate(%s);' % (result, size) +
-                '((pypyhdr_t *)%s)->tid = %s;' % (result, typeid))
+        return '%s = qcgc_allocate(%s);' % (result, size)
+
+    def OP_QCGC_COLLECT(self, op):
+        return 'qcgc_collect();'
diff --git a/rpython/translator/c/src/g_prerequisite.h b/rpython/translator/c/src/g_prerequisite.h
--- a/rpython/translator/c/src/g_prerequisite.h
+++ b/rpython/translator/c/src/g_prerequisite.h
@@ -23,11 +23,3 @@
 # define RPY_LENGTH0     1       /* array decl [0] are bad */
 # define RPY_DUMMY_VARLENGTH     /* nothing */
 #endif
-
-#ifdef RPY_QCGC
-typedef struct {
-	object_t hdr;
-	int32_t tid;
-	int32_t hash;
-} pypyhdr_t;
-#endif

From pypy.commits at gmail.com  Mon Aug 22 10:24:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 07:24:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: import cffi/1362f9f5048a
Message-ID: <57bb0b2f.4152c20a.4c92.86ed@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86419:7a329f45e004
Date: 2016-08-22 16:23 +0200
http://bitbucket.org/pypy/pypy/changeset/7a329f45e004/

Log:	import cffi/1362f9f5048a

diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1963,3 +1963,21 @@
                        ffi, "test_function_returns_opaque", "?")
     assert str(e.value) == ("function foo: 'struct a' is used as result type,"
                             " but is opaque")
+
+def test_function_returns_union():
+    ffi = FFI()
+    ffi.cdef("union u1 { int a, b; }; union u1 f1(int);")
+    lib = verify(ffi, "test_function_returns_union", """
+        union u1 { int a, b; };
+        static union u1 f1(int x) { union u1 u; u.b = x; return u; }
+    """)
+    assert lib.f1(51).a == 51
+
+def test_function_returns_partial_struct():
+    ffi = FFI()
+    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    lib = verify(ffi, "test_function_returns_partial_struct", """
+        struct a { int b, a, c; };
+        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+    """)
+    assert lib.f1(52).a == 52

From pypy.commits at gmail.com  Mon Aug 22 10:25:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 07:25:51 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: update
Message-ID: <57bb0b6f.436ec20a.e855f.829f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5680:7515466117e2
Date: 2016-08-22 16:25 +0200
http://bitbucket.org/pypy/extradoc/changeset/7515466117e2/

Log:	update

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -26,6 +26,12 @@
   ``dict``.  The main pain point is ``move_to_end(last=False)``.  See
   https://mail.python.org/pipermail/python-dev/2016-August/145837.html
 
+* interpreter/generator.py: move the common functionality from
+  GeneratorIterator and Coroutine to the base class.  Review all
+  calls to _PyGen_yf() in genobject.c.  This is needed before
+  adding gi_yieldfrom/cr_await to generator/coroutines.  (Waiting
+  because some work might be going on with raffael_t.)
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------
@@ -48,8 +54,6 @@
 
 * memoryview now supports tuple indexing
 
-* Generators have a new gi_yieldfrom attribute
-
 * A new RecursionError exception is now raised when maximum recursion
   depth is reached. (DONE)
 

From pypy.commits at gmail.com  Mon Aug 22 10:44:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 07:44:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in timfel/pypy (pull request
 #473)
Message-ID: <57bb0fb6.c70a1c0a.bd810.aa60@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r86422:3028b707a5da
Date: 2016-08-22 16:43 +0200
http://bitbucket.org/pypy/pypy/changeset/3028b707a5da/

Log:	Merged in timfel/pypy (pull request #473)

	If code class for vmprof has attrs, extend that

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -90,6 +90,9 @@
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
+        attrs = CodeClass.__dict__.get('_attrs_', None)
+        if attrs is not None:
+            CodeClass._attrs_ = list(attrs) + ['_vmprof_unique_id']
         self._code_classes.add(CodeClass)
         #
         class WeakCodeObjectList(RWeakListMixin):
@@ -189,7 +192,7 @@
 
         def decorated_function(*args):
             unique_id = get_code_fn(*args)._vmprof_unique_id
-            unique_id = rffi.cast(lltype.Signed, unique_id) 
+            unique_id = rffi.cast(lltype.Signed, unique_id)
             # ^^^ removes the "known non-negative" hint for annotation
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)

From pypy.commits at gmail.com  Mon Aug 22 10:44:19 2016
From: pypy.commits at gmail.com (timfel)
Date: Mon, 22 Aug 2016 07:44:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: if code class for vmprof has attrs,
 extend that
Message-ID: <57bb0fc3.94a51c0a.e2706.9bcb@mx.google.com>

Author: Tim Felgentreff <timfelgentreff at gmail.com>
Branch: 
Changeset: r86420:edea3287d97a
Date: 2016-08-22 11:34 +0200
http://bitbucket.org/pypy/pypy/changeset/edea3287d97a/

Log:	if code class for vmprof has attrs, extend that

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -90,6 +90,9 @@
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
+        attrs = CodeClass.__dict__.get('_attrs_', None)
+        if attrs:
+            CodeClass._attrs_ = list(attrs) + ['_vmprof_unique_id']
         self._code_classes.add(CodeClass)
         #
         class WeakCodeObjectList(RWeakListMixin):
@@ -189,7 +192,7 @@
 
         def decorated_function(*args):
             unique_id = get_code_fn(*args)._vmprof_unique_id
-            unique_id = rffi.cast(lltype.Signed, unique_id) 
+            unique_id = rffi.cast(lltype.Signed, unique_id)
             # ^^^ removes the "known non-negative" hint for annotation
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)

From pypy.commits at gmail.com  Mon Aug 22 10:44:21 2016
From: pypy.commits at gmail.com (timfel)
Date: Mon, 22 Aug 2016 07:44:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix attrs test in vmprof for empty
 attrs lists
Message-ID: <57bb0fc5.85261c0a.721b3.a817@mx.google.com>

Author: Tim Felgentreff <timfelgentreff at gmail.com>
Branch: 
Changeset: r86421:6774c47a23c1
Date: 2016-08-22 14:04 +0200
http://bitbucket.org/pypy/pypy/changeset/6774c47a23c1/

Log:	fix attrs test in vmprof for empty attrs lists

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -91,7 +91,7 @@
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
         attrs = CodeClass.__dict__.get('_attrs_', None)
-        if attrs:
+        if attrs is not None:
             CodeClass._attrs_ = list(attrs) + ['_vmprof_unique_id']
         self._code_classes.add(CodeClass)
         #

From pypy.commits at gmail.com  Mon Aug 22 11:30:29 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 22 Aug 2016 08:30:29 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: malloc_fixedsize_clear sets tid
Message-ID: <57bb1a95.031dc20a.34dd5.a926@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86423:f8a7404b3803
Date: 2016-08-22 17:29 +0200
http://bitbucket.org/pypy/pypy/changeset/f8a7404b3803/

Log:	malloc_fixedsize_clear sets tid

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -32,8 +32,12 @@
         ll_assert(not needs_finalizer, 'finalizer not supported')
         ll_assert(not is_finalizer_light, 'light finalizer not supported')
         ll_assert(not contains_weakptr, 'weakref not supported')
-        # FIXME: set typeid and hash here
-        return llop.qcgc_allocate(llmemory.GCREF, size, typeid)
+        raw_mem = llop.qcgc_allocate(llmemory.Address, size, typeid)
+        hdr = llmemory.cast_adr_to_ptr(raw_mem, lltype.Ptr(self.HDR))
+        hdr.tid = rffi.cast(lltype.Signed, typeid)
+        # FIXME: set hash
+        # XXX: Hope the tid/hash setting here is not removed/optimized out
+        return llmemory.cast_adr_to_ptr(raw_mem, llmemory.GCREF)
 
     def malloc_varsize_clear(self, typeid16, length, size, itemsize,
                              offset_to_length):

From pypy.commits at gmail.com  Mon Aug 22 11:40:46 2016
From: pypy.commits at gmail.com (dholth)
Date: Mon, 22 Aug 2016 08:40:46 -0700 (PDT)
Subject: [pypy-commit] cffi default: use py_limited_api flag when available.
Message-ID: <57bb1cfe.85c11c0a.cb5e1.bcda@mx.google.com>

Author: Daniel Holth <dholth at fastmail.fm>
Branch: 
Changeset: r2743:00c0ea3f69a8
Date: 2016-08-21 18:53 -0400
http://bitbucket.org/cffi/cffi/changeset/00c0ea3f69a8/

Log:	use py_limited_api flag when available.

	In setuptools>=26, Extension(..., py_limited_api=True) sets the
	.abi3.so filename, readable by Python 3.2+.

diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py
--- a/cffi/setuptools_ext.py
+++ b/cffi/setuptools_ext.py
@@ -69,16 +69,33 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if Extension.__module__.startswith('setuptools.') and not 'py_limited_api' in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            pass
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/testing/cffi0/test_zintegration.py b/testing/cffi0/test_zintegration.py
--- a/testing/cffi0/test_zintegration.py
+++ b/testing/cffi0/test_zintegration.py
@@ -148,3 +148,28 @@
         p = snip_setuptools_verify2.C.getpwuid(0)
         assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root"
         ''')
+
+    def test_set_py_limited_api(self):
+        from cffi.setuptools_ext import _set_py_limited_api
+        try:
+            import setuptools
+            orig_version = setuptools.__version__
+            setuptools.__version__ = '26.0.0'
+            from setuptools import Extension
+
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+            kwds = _set_py_limited_api(Extension.__base__, {})
+            assert not kwds
+
+            setuptools.__version__ = '25.0'
+            kwds = _set_py_limited_api(Extension, {})
+            assert not kwds
+
+            setuptools.__version__ = 'development'
+            kwds = _set_py_limited_api(Extension, {})
+            assert not kwds
+
+        finally:
+            setuptools.__version__ = orig_version

From pypy.commits at gmail.com  Mon Aug 22 11:40:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 08:40:47 -0700 (PDT)
Subject: [pypy-commit] cffi default: Remove the extra checks,
 which should be unnecessary and possibly cause
Message-ID: <57bb1cff.d41a1c0a.7323d.a1be@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2744:776a85f46170
Date: 2016-08-22 17:26 +0200
http://bitbucket.org/cffi/cffi/changeset/776a85f46170/

Log:	Remove the extra checks, which should be unnecessary and possibly
	cause more confusion.

diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py
--- a/cffi/setuptools_ext.py
+++ b/cffi/setuptools_ext.py
@@ -75,7 +75,7 @@
     Do not alter the setting if it already exists.
     Setuptools takes care of ignoring the flag on Python 2 and PyPy.
     """
-    if Extension.__module__.startswith('setuptools.') and not 'py_limited_api' in kwds:
+    if 'py_limited_api' not in kwds:
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/testing/cffi0/test_zintegration.py b/testing/cffi0/test_zintegration.py
--- a/testing/cffi0/test_zintegration.py
+++ b/testing/cffi0/test_zintegration.py
@@ -160,9 +160,6 @@
             kwds = _set_py_limited_api(Extension, {})
             assert kwds['py_limited_api'] == True
 
-            kwds = _set_py_limited_api(Extension.__base__, {})
-            assert not kwds
-
             setuptools.__version__ = '25.0'
             kwds = _set_py_limited_api(Extension, {})
             assert not kwds

From pypy.commits at gmail.com  Mon Aug 22 11:40:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 08:40:49 -0700 (PDT)
Subject: [pypy-commit] cffi default: If we don't know the version number of
 setuptools, we try to set
Message-ID: <57bb1d01.d32d1c0a.3ca21.b757@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2745:dcb62fcbd63d
Date: 2016-08-22 17:40 +0200
http://bitbucket.org/cffi/cffi/changeset/dcb62fcbd63d/

Log:	If we don't know the version number of setuptools, we try to set
	'py_limited_api' anyway. At worst, we get a warning.

diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py
--- a/cffi/setuptools_ext.py
+++ b/cffi/setuptools_ext.py
@@ -82,7 +82,10 @@
             if setuptools_major_version >= 26:
                 kwds['py_limited_api'] = True
         except ValueError:  # certain development versions of setuptools
-            pass
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
     return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):

From pypy.commits at gmail.com  Mon Aug 22 11:40:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 08:40:51 -0700 (PDT)
Subject: [pypy-commit] cffi default: merge heads
Message-ID: <57bb1d03.c398c20a.80f8e.a98a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2746:3ce478433f9b
Date: 2016-08-22 17:40 +0200
http://bitbucket.org/cffi/cffi/changeset/3ce478433f9b/

Log:	merge heads

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3042,13 +3042,14 @@
 static PyObject *
 convert_struct_to_owning_object(char *data, CTypeDescrObject *ct)
 {
+    /* also accepts unions, for the API mode */
     CDataObject *cd;
     Py_ssize_t dataoffset = offsetof(CDataObject_own_nolength, alignment);
     Py_ssize_t datasize = ct->ct_size;
 
-    if ((ct->ct_flags & (CT_STRUCT|CT_IS_OPAQUE)) != CT_STRUCT) {
+    if (datasize < 0) {
         PyErr_SetString(PyExc_TypeError,
-                        "return type is not a struct or is opaque");
+                        "return type is an opaque structure or union");
         return NULL;
     }
     cd = allocate_owning_object(dataoffset + datasize, ct);
@@ -4623,6 +4624,8 @@
                 ct = ct->ct_itemdescr;
             }
             ffifield = fb_fill_type(fb, ct, 0);
+            if (PyErr_Occurred())
+                return NULL;
             if (elements != NULL) {
                 for (j=0; j<flat; j++)
                     elements[nflat++] = ffifield;
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -269,9 +269,7 @@
 
 Usually, the right thing to do is to call this method with True.  Be
 aware (particularly on Python 2) that, afterwards, you need to pass unicode
-strings as arguments instead of byte strings.  (Before cffi version 0.9,
-``TCHAR`` and friends where hard-coded as unicode, but ``UNICODE`` was,
-inconsistently, not defined by default.)
+strings as arguments instead of byte strings.
 
 
 .. _loading-libraries:
@@ -336,7 +334,7 @@
 
 **ffibuilder.set_source(module_name, c_header_source, [\*\*keywords...])**:
 prepare the ffi for producing out-of-line an external module called
-``module_name``.  *New in version 1.0.*
+``module_name``.
 
 ``ffibuilder.set_source()`` by itself does not write any file, but merely
 records its arguments for later.  It can therefore be called before or
@@ -425,7 +423,7 @@
    declaration which doesn't use "``...``" is assumed to be exact, but this is
    checked: you get an error if it is not correct.
 
-*  *New in version 1.1:* integer types: the syntax "``typedef
+*  integer types: the syntax "``typedef
    int... foo_t;``" declares the type ``foo_t`` as an integer type
    whose exact size and signedness is not specified.  The compiler will
    figure it out.  (Note that this requires ``set_source()``; it does
@@ -462,8 +460,8 @@
    length is completed by the C compiler.
    This is slightly different from "``int n[];``", because the latter
    means that the length is not known even to the C compiler, and thus
-   no attempt is made to complete it.  *New in version 1.1:* support
-   for multidimensional arrays: "``int n[...][...];``".
+   no attempt is made to complete it.  This supports
+   multidimensional arrays: "``int n[...][...];``".
 
    *New in version 1.2:* "``int m[][...];``", i.e. ``...`` can be used
    in the innermost dimensions without being also used in the outermost
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -292,7 +292,7 @@
 
 3. ``ffi.addressof(<library>, "name")`` returns the address of the
 named function or global variable from the given library object.
-*New in version 1.1:* for functions, it returns a regular cdata
+For functions, it returns a regular cdata
 object containing a pointer to the function.
 
 Note that the case 1. cannot be used to take the address of a
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -366,8 +366,8 @@
 
 __ ref.html#conversions
 
-CFFI supports passing and returning structs to functions and callbacks.
-Example:
+CFFI supports passing and returning structs and unions to functions and
+callbacks.  Example:
 
 .. code-block:: python
 
@@ -377,36 +377,33 @@
     myfoo = lib.function_returning_a_struct()
     # `myfoo`: <cdata 'struct foo_s' owning 8 bytes>
 
-There are a few (obscure) limitations to the argument types and return
-type.  You cannot pass directly as argument a union (but a *pointer*
-to a union is fine), nor a struct which uses bitfields (but a
-*pointer* to such a struct is fine).  If you pass a struct (not a
-*pointer* to a struct), the struct type cannot have been declared with
-"``...;``" in the ``cdef()``; you need to declare it completely in
-``cdef()``.  You can work around these limitations by writing a C
-function with a simpler signature in the C header code passed to
-``ffibuilder.set_source()``, and have this C function call the real one.
-
-Aside from these limitations, functions and callbacks can receive and
-return structs.
-
-For performance, API-level functions are not returned as ``<cdata>``
-objects, but as a different type (on CPython, ``<built-in
-function>``).  This means you cannot e.g. pass them to some other C
+For performance, non-variadic API-level functions that you get by
+writing ``lib.some_function`` are not ``<cdata>``
+objects, but an object of a different type (on CPython, ``<built-in
+function>``).  This means you cannot pass them directly to some other C
 function expecting a function pointer argument.  Only ``ffi.typeof()``
 works on them.  To get a cdata containing a regular function pointer,
-use ``ffi.addressof(lib, "name")`` (new in version 1.1).
+use ``ffi.addressof(lib, "name")``.
 
-Before version 1.1 (or with the deprecated ``ffi.verify()``), if you
-really need a cdata pointer to the function, use the following
-workaround:
+There are a few (obscure) limitations to the supported argument and
+return types.  These limitations come from libffi and apply only to
+calling ``<cdata>`` function pointers; in other words, they don't
+apply to non-variadic ``cdef()``-declared functions if you are using
+the API mode.  The limitations are that you cannot pass directly as
+argument or return type:
 
-.. code-block:: python
-  
-    ffi.cdef(""" int (*foo)(int a, int b); """)
+* a union (but a *pointer* to a union is fine);
 
-i.e. declare them as pointer-to-function in the cdef (even if they are
-regular functions in the C code).
+* a struct which uses bitfields (but a *pointer* to such a struct is
+  fine);
+
+* a struct that was declared with "``...``" in the ``cdef()``.
+
+In API mode, you can work around these limitations: for example, if you
+need to call such a function pointer from Python, you can instead write
+a custom C function that accepts the function pointer and the real
+arguments and that does the call from C.  Then declare that custom C
+function in the ``cdef()`` and use it from Python.
 
 
 Variadic function calls
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1962,3 +1962,21 @@
                        ffi, "test_function_returns_opaque", "?")
     assert str(e.value) == ("function foo: 'struct a' is used as result type,"
                             " but is opaque")
+
+def test_function_returns_union():
+    ffi = FFI()
+    ffi.cdef("union u1 { int a, b; }; union u1 f1(int);")
+    lib = verify(ffi, "test_function_returns_union", """
+        union u1 { int a, b; };
+        static union u1 f1(int x) { union u1 u; u.b = x; return u; }
+    """)
+    assert lib.f1(51).a == 51
+
+def test_function_returns_partial_struct():
+    ffi = FFI()
+    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    lib = verify(ffi, "test_function_returns_partial_struct", """
+        struct a { int b, a, c; };
+        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+    """)
+    assert lib.f1(52).a == 52

From pypy.commits at gmail.com  Mon Aug 22 11:45:54 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 22 Aug 2016 08:45:54 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Cleanup,
 set hash and tid from both fixed and varsize
Message-ID: <57bb1e32.898b1c0a.1c93f.ca81@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86424:220e04e78d18
Date: 2016-08-22 17:45 +0200
http://bitbucket.org/pypy/pypy/changeset/220e04e78d18/

Log:	Cleanup, set hash and tid from both fixed and varsize

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -24,6 +24,11 @@
             ('hash', lltype.Signed))
     #HDR = rffi.COpaque('object_t')
 
+    def init_gc_object(self, obj, typeid):
+        hdr = llmemory.cast_adr_to_ptr(obj, lltype.Ptr(self.HDR))
+        hdr.tid = rffi.cast(lltype.Signed, typeid)
+        hdr.hash = rffi.cast(lltype.Signed, obj)
+
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False,
                                is_finalizer_light=False,
@@ -32,18 +37,16 @@
         ll_assert(not needs_finalizer, 'finalizer not supported')
         ll_assert(not is_finalizer_light, 'light finalizer not supported')
         ll_assert(not contains_weakptr, 'weakref not supported')
-        raw_mem = llop.qcgc_allocate(llmemory.Address, size, typeid)
-        hdr = llmemory.cast_adr_to_ptr(raw_mem, lltype.Ptr(self.HDR))
-        hdr.tid = rffi.cast(lltype.Signed, typeid)
-        # FIXME: set hash
-        # XXX: Hope the tid/hash setting here is not removed/optimized out
-        return llmemory.cast_adr_to_ptr(raw_mem, llmemory.GCREF)
+        obj = llop.qcgc_allocate(llmemory.Address, size)
+        self.init_gc_object(obj, typeid)
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
-    def malloc_varsize_clear(self, typeid16, length, size, itemsize,
+    def malloc_varsize_clear(self, typeid, length, size, itemsize,
                              offset_to_length):
         totalsize = size + itemsize * length
         #totalsize = llarena.round_up_for_allocation(totalsize)
-        obj = llop.qcgc_allocate(llmemory.Address, totalsize, typeid16)
+        obj = llop.qcgc_allocate(llmemory.Address, totalsize)
+        self.init_gc_object(obj, typeid)
         (obj + offset_to_length).signed[0] = length
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -68,14 +71,10 @@
     def get_type_id(self, obj):
         return self.header(obj).tid
 
-    def init_gc_object(self, addr, typeid, flags=0):
+    def init_gc_object_immortal(self, addr, typeid, flags=0): # XXX: Prebuilt Objects?
         assert flags == 0
         hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
         hdr.tid = typeid.index
-
-    def init_gc_object_immortal(self, addr, typeid, flags=0): # XXX: Prebuilt Objects?
-        assert flags == 0
-        self.init_gc_object(addr, typeid, flags)
         ptr = self.gcheaderbuilder.object_from_header(addr.ptr)
         prebuilt_hash = lltype.identityhash_nocache(ptr)
         assert prebuilt_hash != 0
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -954,7 +954,6 @@
     def OP_QCGC_ALLOCATE(self, op):
         # XXX: SET typeid
         size = self.expr(op.args[0])
-        typeid = self.expr(op.args[1])
         result = self.expr(op.result)
         return '%s = qcgc_allocate(%s);' % (result, size)
 

From pypy.commits at gmail.com  Mon Aug 22 14:11:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 11:11:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Win32: don't crash translation
Message-ID: <57bb4036.d42f1c0a.b236a.f6e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86425:8e0d3ffa878f
Date: 2016-08-22 20:10 +0200
http://bitbucket.org/pypy/pypy/changeset/8e0d3ffa878f/

Log:	Win32: don't crash translation

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -8,11 +8,13 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.baseobjspace import W_Root
 
-from pypy.module.posix.interp_posix import unwrap_fd, build_stat_result
+from pypy.module.posix.interp_posix import unwrap_fd, build_stat_result, _WIN32
 
 
 def scandir(space, w_path=None):
     "scandir(path='.') -> iterator of DirEntry objects for given path"
+    if _WIN32:
+        raise NotImplementedError("XXX WIN32")
 
     if space.is_none(w_path):
         w_path = space.newunicode(u".")

From pypy.commits at gmail.com  Mon Aug 22 16:03:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 13:03:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: Give the function a name that tells
 which variant it is
Message-ID: <57bb5a98.44ce1c0a.9a4e7.1a2b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86426:d4a1de977834
Date: 2016-08-22 22:03 +0200
http://bitbucket.org/pypy/pypy/changeset/d4a1de977834/

Log:	Give the function a name that tells which variant it is

diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -582,6 +582,14 @@
                 return lltype.malloc(ARRAY, n, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_1_raw_malloc_varsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_1_raw_malloc_varsize.func_name = name
             return _ll_1_raw_malloc_varsize
         return build_ll_1_raw_malloc_varsize
 
@@ -610,6 +618,14 @@
                 return lltype.malloc(STRUCT, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_0_raw_malloc_fixedsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_0_raw_malloc_fixedsize.func_name = name
             return _ll_0_raw_malloc_fixedsize
         return build_ll_0_raw_malloc_fixedsize
 

From pypy.commits at gmail.com  Mon Aug 22 18:40:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 15:40:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57bb7f46.54bc1c0a.66935.556b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86427:7b46c6327f13
Date: 2016-08-22 22:05 +0200
http://bitbucket.org/pypy/pypy/changeset/7b46c6327f13/

Log:	hg merge default

diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -144,3 +144,9 @@
 ``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
 Previously it returned what looked like a regular dict object (but it
 was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1963,3 +1963,21 @@
                        ffi, "test_function_returns_opaque", "?")
     assert str(e.value) == ("function foo: 'struct a' is used as result type,"
                             " but is opaque")
+
+def test_function_returns_union():
+    ffi = FFI()
+    ffi.cdef("union u1 { int a, b; }; union u1 f1(int);")
+    lib = verify(ffi, "test_function_returns_union", """
+        union u1 { int a, b; };
+        static union u1 f1(int x) { union u1 u; u.b = x; return u; }
+    """)
+    assert lib.f1(51).a == 51
+
+def test_function_returns_partial_struct():
+    ffi = FFI()
+    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    lib = verify(ffi, "test_function_returns_partial_struct", """
+        struct a { int b, a, c; };
+        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+    """)
+    assert lib.f1(52).a == 52
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -261,6 +261,9 @@
                    "stack based virtual machines (only for backends that support it)",
                    default=True),
         BoolOption("storesink", "Perform store sinking", default=True),
+        BoolOption("replace_we_are_jitted",
+                   "Replace we_are_jitted() calls by False",
+                   default=False, cmdline=None),
         BoolOption("none",
                    "Do not run any backend optimizations",
                    requires=[('translation.backendopt.inline', False),
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -582,6 +582,14 @@
                 return lltype.malloc(ARRAY, n, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_1_raw_malloc_varsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_1_raw_malloc_varsize.func_name = name
             return _ll_1_raw_malloc_varsize
         return build_ll_1_raw_malloc_varsize
 
@@ -610,6 +618,14 @@
                 return lltype.malloc(STRUCT, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_0_raw_malloc_fixedsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_0_raw_malloc_fixedsize.func_name = name
             return _ll_0_raw_malloc_fixedsize
         return build_ll_0_raw_malloc_fixedsize
 
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -452,7 +452,8 @@
                               merge_if_blocks=True,
                               constfold=True,
                               remove_asserts=True,
-                              really_remove_asserts=True)
+                              really_remove_asserts=True,
+                              replace_we_are_jitted=False)
 
     def prejit_optimizations_minimal_inline(self, policy, graphs):
         from rpython.translator.backendopt.inline import auto_inline_graphs
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -799,8 +799,8 @@
                            rffi.cast(size_t, map_size),
                            rffi.cast(rffi.INT, use_flag))
     else:
-        def madvice_free(addr, map_size):
-            "No madvice() on this platform"
+        def madvise_free(addr, map_size):
+            "No madvise() on this platform"
 
 elif _MS_WINDOWS:
     def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -17,7 +17,7 @@
 def closedir(dirp):
     rposix.c_closedir(dirp)
 
-NULL_DIRP = lltype.nullptr(rposix.DIRENT)
+NULL_DIRP = lltype.nullptr(rposix.DIRP.TO)
 
 def nextentry(dirp):
     """Read the next entry and returns an opaque object.
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -90,6 +90,9 @@
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
+        attrs = CodeClass.__dict__.get('_attrs_', None)
+        if attrs is not None:
+            CodeClass._attrs_ = list(attrs) + ['_vmprof_unique_id']
         self._code_classes.add(CodeClass)
         #
         class WeakCodeObjectList(RWeakListMixin):
@@ -189,7 +192,7 @@
 
         def decorated_function(*args):
             unique_id = get_code_fn(*args)._vmprof_unique_id
-            unique_id = rffi.cast(lltype.Signed, unique_id) 
+            unique_id = rffi.cast(lltype.Signed, unique_id)
             # ^^^ removes the "known non-negative" hint for annotation
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -648,6 +648,7 @@
 
     @staticmethod
     @jit.elidable
+    @signature(types.any(), types.any(), types.int(), types.int(), returns=types.int())
     def ll_rfind_char(s, ch, start, end):
         if end > len(s.chars):
             end = len(s.chars)
diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py
--- a/rpython/translator/backendopt/all.py
+++ b/rpython/translator/backendopt/all.py
@@ -2,6 +2,7 @@
 from rpython.translator.backendopt import inline
 from rpython.translator.backendopt.malloc import remove_mallocs
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.translator.backendopt.stat import print_statistics
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator import simplify
@@ -36,6 +37,7 @@
     # inline_threshold, mallocs
     # merge_if_blocks, constfold, heap2stack
     # clever_malloc_removal, remove_asserts
+    # replace_we_are_jitted
 
     config = translator.config.translation.backendopt.copy(as_default=True)
     config.set(**kwds)
@@ -49,6 +51,10 @@
         print "before optimizations:"
         print_statistics(translator.graphs[0], translator, "per-graph.txt")
 
+    if config.replace_we_are_jitted:
+        for graph in graphs:
+            replace_we_are_jitted(graph)
+
     if config.remove_asserts:
         constfold(config, graphs)
         remove_asserts(translator, graphs)
diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py
--- a/rpython/translator/backendopt/constfold.py
+++ b/rpython/translator/backendopt/constfold.py
@@ -276,3 +276,25 @@
             rewire_links(splitblocks, graph)
         if not diffused and not splitblocks:
             break # finished
+
+def replace_symbolic(graph, symbolic, value):
+    result = False
+    for block in graph.iterblocks():
+        for op in block.operations:
+            for i, arg in enumerate(op.args):
+                if isinstance(arg, Constant) and arg.value is symbolic:
+                    op.args[i] = value
+                    result = True
+        if block.exitswitch is symbolic:
+            block.exitswitch = value
+            result = True
+    return result
+
+def replace_we_are_jitted(graph):
+    from rpython.rlib import jit
+    replacement = Constant(0)
+    replacement.concretetype = lltype.Signed
+    did_replacement = replace_symbolic(graph, jit._we_are_jitted, replacement)
+    if did_replacement:
+        constant_fold_graph(graph)
+    return did_replacement
diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py
--- a/rpython/translator/backendopt/test/test_all.py
+++ b/rpython/translator/backendopt/test/test_all.py
@@ -289,3 +289,19 @@
         llinterp = LLInterpreter(t.rtyper)
         res = llinterp.eval_graph(later_graph, [10])
         assert res == 1
+
+    def test_replace_we_are_jitted(self):
+        from rpython.rlib import jit
+        def f():
+            if jit.we_are_jitted():
+                return 1
+            return 2 + jit.we_are_jitted()
+
+        t = self.translateopt(f, [])
+        graph = graphof(t, f)
+        # by default, replace_we_are_jitted is off
+        assert graph.startblock.operations[0].args[0].value is jit._we_are_jitted
+
+        t = self.translateopt(f, [], replace_we_are_jitted=True)
+        graph = graphof(t, f)
+        assert graph.startblock.exits[0].args[0].value == 2
diff --git a/rpython/translator/backendopt/test/test_constfold.py b/rpython/translator/backendopt/test/test_constfold.py
--- a/rpython/translator/backendopt/test/test_constfold.py
+++ b/rpython/translator/backendopt/test/test_constfold.py
@@ -7,6 +7,7 @@
 from rpython.rtyper import rclass
 from rpython.rlib import objectmodel
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.conftest import option
 
 def get_graph(fn, signature):
@@ -343,3 +344,18 @@
     merge_if_blocks.merge_if_blocks_once(graph)
     constant_fold_graph(graph)
     check_graph(graph, [], 66, t)
+
+def test_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def fn():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+    graph, t = get_graph(fn, [])
+    result = replace_we_are_jitted(graph)
+    assert result
+    checkgraph(graph)
+    # check shape of graph
+    assert len(graph.startblock.operations) == 0
+    assert graph.startblock.exitswitch is None
+    assert graph.startblock.exits[0].target.exits[0].args[0].value == 2
diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -381,7 +381,7 @@
         """ Run all backend optimizations - lltype version
         """
         from rpython.translator.backendopt.all import backend_optimizations
-        backend_optimizations(self.translator)
+        backend_optimizations(self.translator, replace_we_are_jitted=True)
 
 
     STACKCHECKINSERTION = 'stackcheckinsertion_lltype'
diff --git a/rpython/translator/test/test_interactive.py b/rpython/translator/test/test_interactive.py
--- a/rpython/translator/test/test_interactive.py
+++ b/rpython/translator/test/test_interactive.py
@@ -78,3 +78,15 @@
     dll = ctypes.CDLL(str(t.driver.c_entryp))
     f = dll.pypy_g_f
     assert f(2, 3) == 5
+
+def test_check_that_driver_uses_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def f():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+
+    t = Translation(f, [])
+    t.backendopt()
+    graph = t.driver.translator.graphs[0]
+    assert graph.startblock.exits[0].args[0].value == 2

From pypy.commits at gmail.com  Mon Aug 22 18:40:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 15:40:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: JIT bug,
 hard to test for: kill bh_new_raw_buffer() because this single
Message-ID: <57bb7f48.03121c0a.2e400.4d78@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86428:2cc7b34dfe02
Date: 2016-08-23 00:39 +0200
http://bitbucket.org/pypy/pypy/changeset/2cc7b34dfe02/

Log:	JIT bug, hard to test for: kill bh_new_raw_buffer() because this
	single function cannot reproduce the various flags that were given
	to the original raw malloc (zero or not, track allocation or not,
	memory pressure or not). Replaced for now by storing the correct
	function pointer in the VRawBufferInfo instance; this is one of the
	functions '_ll_1_raw_malloc_varsize*' from codewriter.support.

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -833,9 +833,6 @@
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     # vector operations
     vector_arith_code = """
     def bh_vec_{0}_{1}(self, vx, vy, count):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -757,9 +757,6 @@
             self.write_int_at_mem(res, self.vtable_offset, WORD, sizedescr.get_vtable())
         return res
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -229,8 +229,6 @@
         raise NotImplementedError
     def bh_newunicode(self, length):
         raise NotImplementedError
-    def bh_new_raw_buffer(self, size):
-        raise NotImplementedError
 
     def bh_arraylen_gc(self, array, arraydescr):
         raise NotImplementedError
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -367,8 +367,9 @@
 
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
-    
-    def __init__(self, cpu, size=-1):
+
+    def __init__(self, cpu, func, size=-1):
+        self.func = func
         self.size = size
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
@@ -425,7 +426,8 @@
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
         buffer = self._get_buffer()
-        return visitor.visit_vrawbuffer(self.size,
+        return visitor.visit_vrawbuffer(self.func,
+                                        self.size,
                                         buffer.offsets[:],
                                         buffer.descrs[:])
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1770,7 +1770,7 @@
     def test_virtual_raw_malloc_basic(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)   # 12345 = malloc func
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1787,7 +1787,7 @@
         ops = """
         [i1]
         i5 = int_mul(10, 1)
-        i2 = call_i('malloc', i5, descr=raw_malloc_descr)
+        i2 = call_i(12345, i5, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1803,7 +1803,7 @@
     def test_virtual_raw_malloc_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1817,7 +1817,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr_char)
         raw_store(i2, 1, 123, descr=rawarraydescr_char)
@@ -1832,7 +1832,7 @@
     def test_virtual_raw_malloc_invalid_write_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1843,7 +1843,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1855,7 +1855,7 @@
     def test_virtual_raw_malloc_invalid_read_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1866,7 +1866,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr_char)
@@ -1878,7 +1878,7 @@
     def test_virtual_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1898,7 +1898,7 @@
     def test_virtual_raw_slice_of_a_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         i3 = int_add(i2, 1) # get a slice of the original buffer
         i4 = int_add(i3, 1) # get a slice of a slice
@@ -1916,7 +1916,7 @@
     def test_virtual_raw_slice_force(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1929,7 +1929,7 @@
         [i0, i1]
         label(i0, i1)
         # these ops are generated by VirtualRawBufferValue._really_force
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, 42, descr=rawarraydescr_char)
         raw_store(i2, 5, 4242, descr=rawarraydescr_char)
@@ -1946,7 +1946,7 @@
         i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr)
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i3, 0, i2, descr=rawarraydescr)
         label(i2)
@@ -1958,7 +1958,7 @@
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
         label(i2)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i3)
         raw_store(i3, 0, i2, descr=rawarraydescr)
         jump(i3)
@@ -1968,7 +1968,7 @@
     def test_virtual_raw_store_raw_load(self):
         ops = """
         [i1]
-        i0 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i0 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 0, i1, descr=rawarraydescr)
         i2 = raw_load_i(i0, 0, descr=rawarraydescr)
@@ -1986,7 +1986,7 @@
     def test_virtual_raw_store_getarrayitem_raw(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 8, f1, descr=rawarraydescr_float)
         f2 = getarrayitem_raw_f(i0, 1, descr=rawarraydescr_float)
@@ -2004,7 +2004,7 @@
     def test_virtual_setarrayitem_raw_raw_load(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i0, 1, f1, descr=rawarraydescr_float)
         f2 = raw_load_f(i0, 8, descr=rawarraydescr_float)
@@ -2022,7 +2022,7 @@
     def test_virtual_raw_buffer_forced_but_slice_not_forced(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         i1 = int_add(i0, 8)
         escape_n(i0)
@@ -2031,7 +2031,7 @@
         """
         expected = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         check_memory_error(i0)
         escape_n(i0)
         i1 = int_add(i0, 8)
@@ -8886,7 +8886,7 @@
     def test_resume_forced_raw_ptr(self):
         ops = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         is = int_add(i, 8)
         escape_n(i)
@@ -8898,7 +8898,7 @@
         """
         expected = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i)
         escape_n(i)
         i1 = int_add(i0, 1)
@@ -8966,7 +8966,7 @@
     def test_pending_setfield_delayed_malloc(self):
         ops = """
         [i0, p0]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 13, descr=rawarraydescr)
         setfield_gc(p0, i2, descr=valuedescr)
@@ -8988,14 +8988,14 @@
     def test_raw_buffer_ptr_info_intbounds_bug(self):
         ops = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         guard_value(i2, 12345) []
         jump()
         """
         expected = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         guard_value(i2, 12345) []
         jump()
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -45,7 +45,8 @@
         return opinfo
 
     def make_virtual_raw_memory(self, size, source_op):
-        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, size)
+        func = source_op.getarg(0).getint()
+        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, func, size)
         newop = self.replace_op_with(source_op, source_op.getopnum(),
                                      args=[source_op.getarg(0), ConstInt(size)])
         newop.set_forwarded(opinfo)
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -364,8 +364,8 @@
     def visit_varraystruct(self, arraydescr, size, fielddescrs):
         return VArrayStructInfo(arraydescr, size, fielddescrs)
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
-        return VRawBufferInfo(size, offsets, descrs)
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
+        return VRawBufferInfo(func, size, offsets, descrs)
 
     def visit_vrawslice(self, offset):
         return VRawSliceInfo(offset)
@@ -703,7 +703,8 @@
 
 class VRawBufferInfo(VAbstractRawInfo):
 
-    def __init__(self, size, offsets, descrs):
+    def __init__(self, func, size, offsets, descrs):
+        self.func = func
         self.size = size
         self.offsets = offsets
         self.descrs = descrs
@@ -711,7 +712,7 @@
     @specialize.argtype(1)
     def allocate_int(self, decoder, index):
         length = len(self.fieldnums)
-        buffer = decoder.allocate_raw_buffer(self.size)
+        buffer = decoder.allocate_raw_buffer(self.func, self.size)
         decoder.virtuals_cache.set_int(index, buffer)
         for i in range(len(self.offsets)):
             offset = self.offsets[i]
@@ -1130,9 +1131,13 @@
                                                            lengthbox)
         return self.metainterp.execute_new_array(arraydescr, lengthbox)
 
-    def allocate_raw_buffer(self, size):
+    def allocate_raw_buffer(self, func, size):
         cic = self.metainterp.staticdata.callinfocollection
-        calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        # Can't use 'func' from callinfo_for_oopspec(), because we have
+        # several variants (zero/non-zero, memory-pressure or not, etc.)
+        # and we have to pick the correct one here; that's why we save
+        # it in the VRawBufferInfo.
         return self.metainterp.execute_and_record_varargs(
             rop.CALL_I, [ConstInt(func), ConstInt(size)], calldescr)
 
@@ -1461,10 +1466,11 @@
     def allocate_string(self, length):
         return self.cpu.bh_newstr(length)
 
-    def allocate_raw_buffer(self, size):
-        buffer = self.cpu.bh_new_raw_buffer(size)
-        adr = llmemory.cast_ptr_to_adr(buffer)
-        return llmemory.cast_adr_to_int(adr, "symbolic")
+    def allocate_raw_buffer(self, func, size):
+        from rpython.jit.codewriter import heaptracker
+        cic = self.callinfocollection
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        return self.cpu.bh_call_i(func, [size], None, None, calldescr)
 
     def string_setitem(self, str, index, charnum):
         char = self.decode_int(charnum)
diff --git a/rpython/jit/metainterp/walkvirtual.py b/rpython/jit/metainterp/walkvirtual.py
--- a/rpython/jit/metainterp/walkvirtual.py
+++ b/rpython/jit/metainterp/walkvirtual.py
@@ -17,7 +17,7 @@
     def visit_varraystruct(self, arraydescr, fielddescrs):
         raise NotImplementedError("abstract base class")
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
         raise NotImplementedError("abstract base class")
 
     def visit_vrawslice(self, offset):

From pypy.commits at gmail.com  Mon Aug 22 18:40:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 15:40:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57bb7f65.898b1c0a.1c93f.51aa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86429:e9ed73a4cb14
Date: 2016-08-23 00:40 +0200
http://bitbucket.org/pypy/pypy/changeset/e9ed73a4cb14/

Log:	hg merge default

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -833,9 +833,6 @@
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     # vector operations
     vector_arith_code = """
     def bh_vec_{0}_{1}(self, vx, vy, count):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -757,9 +757,6 @@
             self.write_int_at_mem(res, self.vtable_offset, WORD, sizedescr.get_vtable())
         return res
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -229,8 +229,6 @@
         raise NotImplementedError
     def bh_newunicode(self, length):
         raise NotImplementedError
-    def bh_new_raw_buffer(self, size):
-        raise NotImplementedError
 
     def bh_arraylen_gc(self, array, arraydescr):
         raise NotImplementedError
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -367,8 +367,9 @@
 
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
-    
-    def __init__(self, cpu, size=-1):
+
+    def __init__(self, cpu, func, size=-1):
+        self.func = func
         self.size = size
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
@@ -425,7 +426,8 @@
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
         buffer = self._get_buffer()
-        return visitor.visit_vrawbuffer(self.size,
+        return visitor.visit_vrawbuffer(self.func,
+                                        self.size,
                                         buffer.offsets[:],
                                         buffer.descrs[:])
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1770,7 +1770,7 @@
     def test_virtual_raw_malloc_basic(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)   # 12345 = malloc func
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1787,7 +1787,7 @@
         ops = """
         [i1]
         i5 = int_mul(10, 1)
-        i2 = call_i('malloc', i5, descr=raw_malloc_descr)
+        i2 = call_i(12345, i5, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1803,7 +1803,7 @@
     def test_virtual_raw_malloc_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1817,7 +1817,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr_char)
         raw_store(i2, 1, 123, descr=rawarraydescr_char)
@@ -1832,7 +1832,7 @@
     def test_virtual_raw_malloc_invalid_write_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1843,7 +1843,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1855,7 +1855,7 @@
     def test_virtual_raw_malloc_invalid_read_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1866,7 +1866,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr_char)
@@ -1878,7 +1878,7 @@
     def test_virtual_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1898,7 +1898,7 @@
     def test_virtual_raw_slice_of_a_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         i3 = int_add(i2, 1) # get a slice of the original buffer
         i4 = int_add(i3, 1) # get a slice of a slice
@@ -1916,7 +1916,7 @@
     def test_virtual_raw_slice_force(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1929,7 +1929,7 @@
         [i0, i1]
         label(i0, i1)
         # these ops are generated by VirtualRawBufferValue._really_force
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, 42, descr=rawarraydescr_char)
         raw_store(i2, 5, 4242, descr=rawarraydescr_char)
@@ -1946,7 +1946,7 @@
         i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr)
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i3, 0, i2, descr=rawarraydescr)
         label(i2)
@@ -1958,7 +1958,7 @@
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
         label(i2)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i3)
         raw_store(i3, 0, i2, descr=rawarraydescr)
         jump(i3)
@@ -1968,7 +1968,7 @@
     def test_virtual_raw_store_raw_load(self):
         ops = """
         [i1]
-        i0 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i0 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 0, i1, descr=rawarraydescr)
         i2 = raw_load_i(i0, 0, descr=rawarraydescr)
@@ -1986,7 +1986,7 @@
     def test_virtual_raw_store_getarrayitem_raw(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 8, f1, descr=rawarraydescr_float)
         f2 = getarrayitem_raw_f(i0, 1, descr=rawarraydescr_float)
@@ -2004,7 +2004,7 @@
     def test_virtual_setarrayitem_raw_raw_load(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i0, 1, f1, descr=rawarraydescr_float)
         f2 = raw_load_f(i0, 8, descr=rawarraydescr_float)
@@ -2022,7 +2022,7 @@
     def test_virtual_raw_buffer_forced_but_slice_not_forced(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         i1 = int_add(i0, 8)
         escape_n(i0)
@@ -2031,7 +2031,7 @@
         """
         expected = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         check_memory_error(i0)
         escape_n(i0)
         i1 = int_add(i0, 8)
@@ -8886,7 +8886,7 @@
     def test_resume_forced_raw_ptr(self):
         ops = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         is = int_add(i, 8)
         escape_n(i)
@@ -8898,7 +8898,7 @@
         """
         expected = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i)
         escape_n(i)
         i1 = int_add(i0, 1)
@@ -8966,7 +8966,7 @@
     def test_pending_setfield_delayed_malloc(self):
         ops = """
         [i0, p0]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 13, descr=rawarraydescr)
         setfield_gc(p0, i2, descr=valuedescr)
@@ -8988,14 +8988,14 @@
     def test_raw_buffer_ptr_info_intbounds_bug(self):
         ops = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         guard_value(i2, 12345) []
         jump()
         """
         expected = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         guard_value(i2, 12345) []
         jump()
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -45,7 +45,8 @@
         return opinfo
 
     def make_virtual_raw_memory(self, size, source_op):
-        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, size)
+        func = source_op.getarg(0).getint()
+        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, func, size)
         newop = self.replace_op_with(source_op, source_op.getopnum(),
                                      args=[source_op.getarg(0), ConstInt(size)])
         newop.set_forwarded(opinfo)
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -364,8 +364,8 @@
     def visit_varraystruct(self, arraydescr, size, fielddescrs):
         return VArrayStructInfo(arraydescr, size, fielddescrs)
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
-        return VRawBufferInfo(size, offsets, descrs)
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
+        return VRawBufferInfo(func, size, offsets, descrs)
 
     def visit_vrawslice(self, offset):
         return VRawSliceInfo(offset)
@@ -703,7 +703,8 @@
 
 class VRawBufferInfo(VAbstractRawInfo):
 
-    def __init__(self, size, offsets, descrs):
+    def __init__(self, func, size, offsets, descrs):
+        self.func = func
         self.size = size
         self.offsets = offsets
         self.descrs = descrs
@@ -711,7 +712,7 @@
     @specialize.argtype(1)
     def allocate_int(self, decoder, index):
         length = len(self.fieldnums)
-        buffer = decoder.allocate_raw_buffer(self.size)
+        buffer = decoder.allocate_raw_buffer(self.func, self.size)
         decoder.virtuals_cache.set_int(index, buffer)
         for i in range(len(self.offsets)):
             offset = self.offsets[i]
@@ -1130,9 +1131,13 @@
                                                            lengthbox)
         return self.metainterp.execute_new_array(arraydescr, lengthbox)
 
-    def allocate_raw_buffer(self, size):
+    def allocate_raw_buffer(self, func, size):
         cic = self.metainterp.staticdata.callinfocollection
-        calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        # Can't use 'func' from callinfo_for_oopspec(), because we have
+        # several variants (zero/non-zero, memory-pressure or not, etc.)
+        # and we have to pick the correct one here; that's why we save
+        # it in the VRawBufferInfo.
         return self.metainterp.execute_and_record_varargs(
             rop.CALL_I, [ConstInt(func), ConstInt(size)], calldescr)
 
@@ -1461,10 +1466,11 @@
     def allocate_string(self, length):
         return self.cpu.bh_newstr(length)
 
-    def allocate_raw_buffer(self, size):
-        buffer = self.cpu.bh_new_raw_buffer(size)
-        adr = llmemory.cast_ptr_to_adr(buffer)
-        return llmemory.cast_adr_to_int(adr, "symbolic")
+    def allocate_raw_buffer(self, func, size):
+        from rpython.jit.codewriter import heaptracker
+        cic = self.callinfocollection
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        return self.cpu.bh_call_i(func, [size], None, None, calldescr)
 
     def string_setitem(self, str, index, charnum):
         char = self.decode_int(charnum)
diff --git a/rpython/jit/metainterp/walkvirtual.py b/rpython/jit/metainterp/walkvirtual.py
--- a/rpython/jit/metainterp/walkvirtual.py
+++ b/rpython/jit/metainterp/walkvirtual.py
@@ -17,7 +17,7 @@
     def visit_varraystruct(self, arraydescr, fielddescrs):
         raise NotImplementedError("abstract base class")
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
         raise NotImplementedError("abstract base class")
 
     def visit_vrawslice(self, offset):

From pypy.commits at gmail.com  Mon Aug 22 19:01:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 Aug 2016 16:01:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix test
Message-ID: <57bb8438.11051c0a.b5f3a.4cff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86430:f8852bc52436
Date: 2016-08-23 01:00 +0200
http://bitbucket.org/pypy/pypy/changeset/f8852bc52436/

Log:	Fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -423,7 +423,7 @@
         guard_false(i114, descr=...)
         --TICK--
         i123 = arraylen_gc(p67, descr=<ArrayP .>)
-        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=5 OS=110>)
+        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=<Calli . i EF=5 OS=110>)
         check_memory_error(i119)
         raw_store(i119, 0, i160, descr=<ArrayS 2>)
         raw_store(i119, 2, i160, descr=<ArrayS 2>)

From pypy.commits at gmail.com  Tue Aug 23 00:20:00 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 22 Aug 2016 21:20:00 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: fix default value
Message-ID: <57bbcef0.03121c0a.2e400.9169@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86431:6d9a8c05c0e2
Date: 2016-08-23 14:19 +1000
http://bitbucket.org/pypy/pypy/changeset/6d9a8c05c0e2/

Log:	fix default value

diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -468,7 +468,8 @@
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_buffer = space.call_method(w_buffer, '__buffer__', space.wrap(0))
+        w_buffer = space.call_method(w_buffer, '__buffer__', 
+                                    space.newint(space.BUF_FULL_RO))
         buf = _getbuffer(space, w_buffer)
 
     ts = buf.getlength()

From pypy.commits at gmail.com  Tue Aug 23 01:18:48 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 22 Aug 2016 22:18:48 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: newbool requires a bool,
 not int
Message-ID: <57bbdcb8.041f1c0a.6f5fb.a0c3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86432:9a44d04048ef
Date: 2016-08-23 15:17 +1000
http://bitbucket.org/pypy/pypy/changeset/9a44d04048ef/

Log:	newbool requires a bool, not int

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -127,7 +127,7 @@
         return space.wrap(self.buf.getndim())
 
     def w_is_readonly(self, space):
-        return space.newbool(self.buf.readonly)
+        return space.newbool(self.buf.readonly != 0)
 
     def w_get_shape(self, space):
         return space.newtuple([space.wrap(x) for x in self.buf.getshape()])

From pypy.commits at gmail.com  Tue Aug 23 01:38:36 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 22 Aug 2016 22:38:36 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Add FIXME for bug in trace
 function
Message-ID: <57bbe15c.482cc20a.e404.6a3e@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86433:5814b9e70ba5
Date: 2016-08-23 07:37 +0200
http://bitbucket.org/pypy/pypy/changeset/5814b9e70ba5/

Log:	Add FIXME for bug in trace function

diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -22,7 +22,7 @@
         #     s_gcref], s_gcref)
         #
         def invokecallback(root, visit_fn):
-            visit_fn(root)
+            visit_fn(root) # FIXME: Dereference root before invoking visit_fn
         def pypy_trace_cb(obj, visit_fn):
             gc.trace(obj, invokecallback, visit_fn)
         pypy_trace_cb.c_name = "pypy_trace_cb"

From pypy.commits at gmail.com  Tue Aug 23 02:35:36 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 22 Aug 2016 23:35:36 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: revert to original return
 value
Message-ID: <57bbeeb8.43681c0a.9fdef.a8ea@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: memoryview-attributes
Changeset: r86434:f3bd9cd817f0
Date: 2016-08-23 16:34 +1000
http://bitbucket.org/pypy/pypy/changeset/f3bd9cd817f0/

Log:	revert to original return value

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -464,8 +464,11 @@
                     "Cannot use string as modifiable buffer")
 
     def descr_getbuffer(self, space, w_flags):
-        from pypy.objspace.std.bufferobject import W_Buffer
-        return W_Buffer(StringBuffer(self._value))
+        #from pypy.objspace.std.bufferobject import W_Buffer
+        #return W_Buffer(StringBuffer(self._value))
+        # XXX handle flags, figure out why returning a W_Buffer
+        #     makes upstream ctypes tests fail
+        return self
 
     charbuf_w = str_w
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -127,7 +127,7 @@
         return space.wrap(self.buf.getndim())
 
     def w_is_readonly(self, space):
-        return space.newbool(self.buf.readonly != 0)
+        return space.newbool(bool(self.buf.readonly))
 
     def w_get_shape(self, space):
         return space.newtuple([space.wrap(x) for x in self.buf.getshape()])

From pypy.commits at gmail.com  Tue Aug 23 02:50:20 2016
From: pypy.commits at gmail.com (wlav)
Date: Mon, 22 Aug 2016 23:50:20 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: complete fast path
Message-ID: <57bbf22c.47cbc20a.e561a.7bed@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86435:0b1eabb9b806
Date: 2016-08-22 21:46 -0700
http://bitbucket.org/pypy/pypy/changeset/0b1eabb9b806/

Log:	complete fast path

diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -8,6 +8,8 @@
 from pypy.interpreter.error import oefmt
 
 from pypy.module._cffi_backend import ctypefunc, ctypeprim, cdataobj, misc
+from pypy.module._cffi_backend import newtype
+from pypy.module.cppyy import ffitypes
 
 from pypy.module.cppyy.capi.capi_types import C_SCOPE, C_TYPE, C_OBJECT,\
    C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX, C_FUNC_PTR
@@ -90,35 +92,35 @@
         self.library = None
         self.capi_calls = {}
 
-        import pypy.module._cffi_backend.newtype as nt
+        nt = newtype     # module from _cffi_backend
+        state = space.fromcache(ffitypes.State)   # factored out common types
 
         # TODO: the following need to match up with the globally defined C_XYZ low-level
         # types (see capi/__init__.py), but by using strings here, that isn't guaranteed
-        c_opaque_ptr = nt.new_primitive_type(space, 'unsigned long')
+        c_opaque_ptr = state.c_ulong
  
         c_scope  = c_opaque_ptr
         c_type   = c_scope
         c_object = c_opaque_ptr
         c_method = c_opaque_ptr
-        c_index  = nt.new_primitive_type(space, 'long')
+        c_index  = state.c_long
+        c_index_array = state.c_voidp
 
-        c_void   = nt.new_void_type(space)
-        c_char   = nt.new_primitive_type(space, 'char')
-        c_uchar  = nt.new_primitive_type(space, 'unsigned char')
-        c_short  = nt.new_primitive_type(space, 'short')
-        c_int    = nt.new_primitive_type(space, 'int')
-        c_long   = nt.new_primitive_type(space, 'long')
-        c_llong  = nt.new_primitive_type(space, 'long long')
-        c_ullong = nt.new_primitive_type(space, 'unsigned long long')
-        c_float  = nt.new_primitive_type(space, 'float')
-        c_double = nt.new_primitive_type(space, 'double')
+        c_void   = state.c_void
+        c_char   = state.c_char
+        c_uchar  = state.c_uchar
+        c_short  = state.c_short
+        c_int    = state.c_int
+        c_long   = state.c_long
+        c_llong  = state.c_llong
+        c_ullong = state.c_ullong
+        c_float  = state.c_float
+        c_double = state.c_double
 
-        c_ccharp = nt.new_pointer_type(space, c_char)
-        c_index_array = nt.new_pointer_type(space, c_void)
+        c_ccharp = state.c_ccharp
+        c_voidp  = state.c_voidp
 
-        c_voidp  = nt.new_pointer_type(space, c_void)
         c_size_t = nt.new_primitive_type(space, 'size_t')
-
         c_ptrdiff_t = nt.new_primitive_type(space, 'ptrdiff_t')
 
         self.capi_call_ifaces = {
diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -4,7 +4,7 @@
 
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import r_singlefloat
-from rpython.rlib import jit_libffi, rfloat
+from rpython.rlib import rfloat
 
 from pypy.module._rawffi.interp_rawffi import letter2tp
 from pypy.module._rawffi.array import W_Array, W_ArrayInstance
@@ -81,11 +81,11 @@
 
 
 class TypeConverter(object):
-    _immutable_fields_ = ['libffitype', 'uses_local', 'name']
+    _immutable_fields_ = ['cffi_name', 'uses_local', 'name']
 
-    libffitype = lltype.nullptr(jit_libffi.FFI_TYPE_P.TO)
+    cffi_name  = None
     uses_local = False
-    name = ""
+    name       = ""
 
     def __init__(self, space, extra):
         pass
@@ -103,6 +103,10 @@
         raise oefmt(space.w_TypeError,
                     "no converter available for '%s'", self.name)
 
+    def cffi_type(self, space):
+        from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
+        raise FastCallNotPossible
+
     def convert_argument(self, space, w_obj, address, call_local):
         self._is_abstract(space)
 
@@ -143,9 +147,7 @@
 
 class ArrayTypeConverterMixin(object):
     _mixin_ = True
-    _immutable_fields_ = ['libffitype', 'size']
-
-    libffitype = jit_libffi.types.pointer
+    _immutable_fields_ = ['size']
 
     def __init__(self, space, array_size):
         if array_size <= 0:
@@ -153,6 +155,10 @@
         else:
             self.size = array_size
 
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
@@ -172,13 +178,15 @@
 
 class PtrTypeConverterMixin(object):
     _mixin_ = True
-    _immutable_fields_ = ['libffitype', 'size']
-
-    libffitype = jit_libffi.types.pointer
+    _immutable_fields_ = ['size']
 
     def __init__(self, space, array_size):
         self.size = sys.maxint
 
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
     def convert_argument(self, space, w_obj, address, call_local):
         w_tc = space.findattr(w_obj, space.wrap('typecode'))
         if w_tc is not None and space.str_w(w_tc) != self.typecode:
@@ -241,6 +249,10 @@
 
     uses_local = True
 
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
     def convert_argument_libffi(self, space, w_obj, address, call_local):
         assert rffi.sizeof(self.c_type) <= 2*rffi.sizeof(rffi.VOIDP)  # see interp_cppyy.py
         obj = self._unwrap_object(space, w_obj)
@@ -269,13 +281,15 @@
 
 
 class VoidConverter(TypeConverter):
-    _immutable_fields_ = ['libffitype', 'name']
-
-    libffitype = jit_libffi.types.void
+    _immutable_fields_ = ['name']
 
     def __init__(self, space, name):
         self.name = name
 
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_void
+
     def convert_argument(self, space, w_obj, address, call_local):
         self._is_abstract(space)
 
@@ -340,10 +354,12 @@
         return space.wrap(float(rffiptr[0]))
 
 class ConstFloatRefConverter(FloatConverter):
-    _immutable_fields_ = ['libffitype', 'typecode']
+    _immutable_fields_ = ['typecode']
+    typecode = 'f'
 
-    libffitype = jit_libffi.types.pointer
-    typecode = 'f'
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
 
     def convert_argument_libffi(self, space, w_obj, address, call_local):
         from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
@@ -359,12 +375,9 @@
             self.default = rffi.cast(self.c_type, 0.)
 
 class ConstDoubleRefConverter(ConstRefNumericTypeConverterMixin, DoubleConverter):
-    _immutable_fields_ = ['libffitype', 'typecode']
-
-    libffitype = jit_libffi.types.pointer
+    _immutable_fields_ = ['typecode']
     typecode = 'd'
 
-
 class CStringConverter(TypeConverter):
     def convert_argument(self, space, w_obj, address, call_local):
         x = rffi.cast(rffi.LONGP, address)
@@ -383,10 +396,6 @@
 
 
 class VoidPtrConverter(TypeConverter):
-    _immutable_fields_ = ['libffitype']
-
-    libffitype = jit_libffi.types.pointer
-
     def _unwrap_object(self, space, w_obj):
         try:
             obj = get_rawbuffer(space, w_obj)
@@ -399,6 +408,10 @@
                 obj = rffi.cast(rffi.VOIDP, get_rawobject(space, w_obj))
         return obj
 
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
     def convert_argument(self, space, w_obj, address, call_local):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
@@ -457,9 +470,7 @@
     typecode   = 'V'
 
 class InstanceRefConverter(TypeConverter):
-    _immutable_fields_ = ['libffitype', 'typecode', 'cppclass']
-
-    libffitype  = jit_libffi.types.pointer
+    _immutable_fields_ = ['typecode', 'cppclass']
     typecode    = 'V'
 
     def __init__(self, space, cppclass):
@@ -478,6 +489,10 @@
         raise oefmt(space.w_TypeError,
                     "cannot pass %T as %s", w_obj, self.cppclass.name)
 
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
     def convert_argument(self, space, w_obj, address, call_local):
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_obj))
@@ -596,9 +611,9 @@
 
 
 class PyObjectConverter(TypeConverter):
-    _immutable_fields_ = ['libffitype']
-
-    libffitype = jit_libffi.types.pointer
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
 
     def convert_argument(self, space, w_obj, address, call_local):
         if hasattr(space, "fake"):
@@ -745,7 +760,6 @@
                 self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
-            libffitype = jit_libffi.types.pointer
         for name in names:
             _converters[name] = BasicConverter
             _converters["const "+name+"&"] = ConstRefConverter
@@ -763,7 +777,6 @@
                 self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
-            libffitype = jit_libffi.types.pointer
         for name in names:
             _converters[name] = BasicConverter
             _converters["const "+name+"&"] = ConstRefConverter
@@ -784,7 +797,6 @@
                 self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
-            libffitype = jit_libffi.types.pointer
         for name in names:
             _converters[name] = BasicConverter
             _converters["const "+name+"&"] = ConstRefConverter
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -27,13 +27,13 @@
 NULL = lltype.nullptr(jit_libffi.FFI_TYPE_P.TO)
 
 class FunctionExecutor(object):
-    _immutable_fields_ = ['libffitype']
-
-    libffitype = NULL
-
     def __init__(self, space, extra):
         pass
 
+    def cffi_type(self, space):
+        from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
+        raise FastCallNotPossible
+
     def execute(self, space, cppmethod, cppthis, num_args, args):
         raise oefmt(space.w_TypeError,
                     "return type not available or supported")
@@ -44,10 +44,12 @@
 
 
 class PtrTypeExecutor(FunctionExecutor):
-    _immutable_fields_ = ['libffitype', 'typecode']
+    _immutable_fields_ = ['typecode']
+    typecode = 'P'
 
-    libffitype = jit_libffi.types.pointer
-    typecode = 'P'
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         if hasattr(space, "fake"):
@@ -62,9 +64,9 @@
 
 
 class VoidExecutor(FunctionExecutor):
-    _immutable_fields_ = ['libffitype']
-
-    libffitype = jit_libffi.types.void
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_void
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         capi.c_call_v(space, cppmethod, cppthis, num_args, args)
@@ -143,14 +145,16 @@
 
 
 class InstancePtrExecutor(FunctionExecutor):
-    _immutable_fields_ = ['libffitype', 'cppclass']
-
-    libffitype = jit_libffi.types.pointer
+    _immutable_fields_ = ['cppclass']
 
     def __init__(self, space, cppclass):
         FunctionExecutor.__init__(self, space, cppclass)
         self.cppclass = cppclass
 
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
     def execute(self, space, cppmethod, cppthis, num_args, args):
         from pypy.module.cppyy import interp_cppyy
         long_result = capi.c_call_l(space, cppmethod, cppthis, num_args, args)
@@ -331,8 +335,9 @@
             _immutable_ = True
             c_stubcall  = staticmethod(stub)
         class BasicRefExecutor(ffitypes.typeid(c_type), NumericRefExecutorMixin, FunctionExecutor):
-            _immutable_fields_ = ['libffitype']
-            libffitype = jit_libffi.types.pointer
+            def cffi_type(self, space):
+                state = space.fromcache(ffitypes.State)
+                return state.c_voidp
         for name in names:
             _executors[name]              = BasicExecutor
             _executors[name+'&']          = BasicRefExecutor
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -2,19 +2,50 @@
 
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rarithmetic import r_singlefloat
-from rpython.rlib import jit_libffi, rfloat
 
-# Mixins to share between converter and executor classes (in converter.py and
-# executor.py, respectively). Basically these mixins allow grouping of the
-# sets of jit_libffi, rffi, and different space unwrapping calls. To get the
-# right mixin, a non-RPython function typeid() is used.
+from pypy.module._cffi_backend import newtype
+
+# Mixins to refactor type-specific codef from converter and executor classes
+# (in converter.py and executor.py, respectively). To get the right mixin, a
+# non-RPython function typeid() is used.
+
+class State(object):
+    def __init__(self, space):
+        self.library = None
+        self.capi_calls = {}
+
+        nt = newtype     # module from _cffi_backend
+
+        # builtin types
+        self.c_void    = nt.new_void_type(space)
+        self.c_bool    = nt.new_primitive_type(space, '_Bool')
+        self.c_char    = nt.new_primitive_type(space, 'char')
+        self.c_uchar   = nt.new_primitive_type(space, 'unsigned char')
+        self.c_short   = nt.new_primitive_type(space, 'short')
+        self.c_ushort  = nt.new_primitive_type(space, 'unsigned short')
+        self.c_int     = nt.new_primitive_type(space, 'int')
+        self.c_uint    = nt.new_primitive_type(space, 'unsigned int')
+        self.c_long    = nt.new_primitive_type(space, 'long')
+        self.c_ulong   = nt.new_primitive_type(space, 'unsigned long')
+        self.c_llong   = nt.new_primitive_type(space, 'long long')
+        self.c_ullong  = nt.new_primitive_type(space, 'unsigned long long')
+        self.c_float   = nt.new_primitive_type(space, 'float')
+        self.c_double  = nt.new_primitive_type(space, 'double')
+        self.c_ldouble = nt.new_primitive_type(space, 'long double')
+        
+        # pointer types
+        self.c_ccharp = nt.new_pointer_type(space, self.c_char)
+        self.c_voidp  = nt.new_pointer_type(space, self.c_void)
+
+        # special types
+        self.c_size_t    = nt.new_primitive_type(space, 'size_t')
+        self.c_ptrdiff_t = nt.new_primitive_type(space, 'ptrdiff_t')
 
 
 class BoolTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.uchar
     c_type      = rffi.UCHAR
     c_ptrtype   = rffi.UCHARP
 
@@ -28,11 +59,14 @@
     def _wrap_object(self, space, obj):
         return space.wrap(bool(ord(rffi.cast(rffi.CHAR, obj))))
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_bool
+
 class CharTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.schar
     c_type      = rffi.CHAR
     c_ptrtype   = rffi.CCHARP           # there's no such thing as rffi.CHARP
 
@@ -52,100 +86,126 @@
                         "char expected, got string of size %d", len(value))
         return value[0] # turn it into a "char" to the annotator
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_char
+
 class ShortTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.sshort
     c_type      = rffi.SHORT
     c_ptrtype   = rffi.SHORTP
 
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(rffi.SHORT, space.int_w(w_obj))
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_short
+
 class UShortTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.ushort
     c_type      = rffi.USHORT
     c_ptrtype   = rffi.USHORTP
 
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(self.c_type, space.int_w(w_obj))
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_ushort
+
 class IntTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.sint
     c_type      = rffi.INT
     c_ptrtype   = rffi.INTP
-    ctype_name  = 'int'
 
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(self.c_type, space.c_int_w(w_obj))
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_int
+
 class UIntTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.uint
     c_type      = rffi.UINT
     c_ptrtype   = rffi.UINTP
 
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(self.c_type, space.uint_w(w_obj))
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_uint
+
 class LongTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.slong
     c_type      = rffi.LONG
     c_ptrtype   = rffi.LONGP
 
     def _unwrap_object(self, space, w_obj):
         return space.int_w(w_obj)
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_long
+
 class ULongTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.ulong
     c_type      = rffi.ULONG
     c_ptrtype   = rffi.ULONGP
 
     def _unwrap_object(self, space, w_obj):
         return space.uint_w(w_obj)
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_ulong
+
 class LongLongTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.sint64
     c_type      = rffi.LONGLONG
     c_ptrtype   = rffi.LONGLONGP
 
     def _unwrap_object(self, space, w_obj):
         return space.r_longlong_w(w_obj)
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_llong
+
 class ULongLongTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
 
-    libffitype  = jit_libffi.types.uint64
     c_type      = rffi.ULONGLONG
     c_ptrtype   = rffi.ULONGLONGP
 
     def _unwrap_object(self, space, w_obj):
         return space.r_ulonglong_w(w_obj)
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_ullong
+
 class FloatTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype', 'typecode']
+    _immutable_fields_ = ['c_type', 'c_ptrtype', 'typecode']
 
-    libffitype  = jit_libffi.types.float
     c_type      = rffi.FLOAT
     c_ptrtype   = rffi.FLOATP
     typecode    = 'f'
@@ -156,11 +216,14 @@
     def _wrap_object(self, space, obj):
         return space.wrap(float(obj))
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_float
+
 class DoubleTypeMixin(object):
     _mixin_     = True
-    _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype', 'typecode']
+    _immutable_fields_ = ['c_type', 'c_ptrtype', 'typecode']
 
-    libffitype  = jit_libffi.types.double
     c_type      = rffi.DOUBLE
     c_ptrtype   = rffi.DOUBLEP
     typecode    = 'd'
@@ -168,6 +231,24 @@
     def _unwrap_object(self, space, w_obj):
         return space.float_w(w_obj)
 
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_double
+
+class LongDoubleTypeMixin(object):
+    _mixin_     = True
+    _immutable_fields_ = ['c_type', 'c_ptrtype', 'typecode']
+
+    c_type      = rffi.LONGDOUBLE
+    c_ptrtype   = rffi.LONGDOUBLEP
+    typecode    = 'D'
+
+    def _unwrap_object(self, space, w_obj):
+        return space.float_w(w_obj)
+
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_ldouble
 
 def typeid(c_type):
     "NOT_RPYTHON"
@@ -183,6 +264,7 @@
     if c_type == rffi.ULONGLONG:  return ULongLongTypeMixin
     if c_type == rffi.FLOAT:      return FloatTypeMixin
     if c_type == rffi.DOUBLE:     return DoubleTypeMixin
+    if c_type == rffi.LONGDOUBLE: return LongDoubleTypeMixin
 
     # should never get here
     raise TypeError("unknown rffi type: %s" % c_type)
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -11,8 +11,8 @@
 from rpython.rlib import jit_libffi, clibffi
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 
-from pypy.module._cffi_backend import ctypefunc, newtype
-from pypy.module.cppyy import converter, executor, helper
+from pypy.module._cffi_backend import ctypefunc
+from pypy.module.cppyy import converter, executor, ffitypes, helper
 
 
 class FastCallNotPossible(Exception):
@@ -209,7 +209,7 @@
         if self.converters is None:
             try:
                 self._setup(cppthis)
-            except Exception as e:
+            except Exception:
                 pass
 
         # some calls, e.g. for ptr-ptr or reference need a local array to store data for
@@ -261,6 +261,7 @@
                 data = capi.exchange_address(buffer, cif_descr, j+1)
                 conv.default_argument_libffi(self.space, data)
 
+            assert self._funcaddr
             w_res = self.executor.execute_libffi(
                 self.space, cif_descr, self._funcaddr, buffer)
         finally:
@@ -326,14 +327,17 @@
         # uniquely defined and needs to be setup only once.
         self._funcaddr = capi.c_get_function_address(self.space, self.scope, self.index)
         if self._funcaddr and cppthis:      # methods only for now
+            state = self.space.fromcache(ffitypes.State)
+
             # argument type specification (incl. cppthis)
             fargs = []
-            fargs.append(newtype.new_pointer_type(self.space, newtype.new_void_type(self.space)))
-            for i, conv in enumerate(self.converters):
-                 if not conv.libffitype:
-                     raise FastCallNotPossible
-                 fargs.append(newtype.new_primitive_type(self.space, conv.ctype_name))
-            fresult = newtype.new_primitive_type(self.space, self.executor.ctype_name)
+            try:
+                fargs.append(state.c_voidp)
+                for i, conv in enumerate(self.converters):
+                    fargs.append(conv.cffi_type(self.space))
+                fresult = self.executor.cffi_type(self.space)
+            except:
+                raise FastCallNotPossible
 
             # the following is derived from _cffi_backend.ctypefunc
             builder = ctypefunc.CifDescrBuilder(fargs[:], fresult, clibffi.FFI_DEFAULT_ABI)
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -89,9 +89,10 @@
     def get_raw_address(self):
         raise ValueError("no raw buffer")
 class FakeException(FakeType):
-    def __init__(self, name):
+    def __init__(self, space, name):
         FakeType.__init__(self, name)
         self.msg = name
+        self.space = space
 
 class FakeUserDelAction(object):
     def __init__(self, space):
@@ -110,15 +111,6 @@
 class FakeSpace(object):
     fake = True
 
-    w_AttributeError      = FakeException("AttributeError")
-    w_KeyError            = FakeException("KeyError")
-    w_NotImplementedError = FakeException("NotImplementedError")
-    w_ReferenceError      = FakeException("ReferenceError")
-    w_RuntimeError        = FakeException("RuntimeError")
-    w_SystemError         = FakeException("SystemError")
-    w_TypeError           = FakeException("TypeError")
-    w_ValueError          = FakeException("ValueError")
-
     w_None = None
     w_str = FakeType("str")
     w_int = FakeType("int")
@@ -137,6 +129,15 @@
             return capi.c_call_i(space, cppmethod, cppobject, nargs, args)
         executor.get_executor(self, 'int').__class__.c_stubcall = staticmethod(c_call_i)
 
+        self.w_AttributeError      = FakeException(self, "AttributeError")
+        self.w_KeyError            = FakeException(self, "KeyError")
+        self.w_NotImplementedError = FakeException(self, "NotImplementedError")
+        self.w_ReferenceError      = FakeException(self, "ReferenceError")
+        self.w_RuntimeError        = FakeException(self, "RuntimeError")
+        self.w_SystemError         = FakeException(self, "SystemError")
+        self.w_TypeError           = FakeException(self, "TypeError")
+        self.w_ValueError          = FakeException(self, "ValueError")
+
     def issequence_w(self, w_obj):
         return True
 

From pypy.commits at gmail.com  Tue Aug 23 02:50:22 2016
From: pypy.commits at gmail.com (wlav)
Date: Mon, 22 Aug 2016 23:50:22 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: add offset elision for simple
 hierarchies
Message-ID: <57bbf22e.c19d1c0a.a7ff3.bcb5@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r86436:07248074d30c
Date: 2016-08-22 22:00 -0700
http://bitbucket.org/pypy/pypy/changeset/07248074d30c/

Log:	add offset elision for simple hierarchies

diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -627,10 +627,28 @@
    return cr->GetName();
 }
 
-Bool_t Cppyy::HasComplexHierarchy( TCppType_t /* handle */ )
+Bool_t Cppyy::HasComplexHierarchy( TCppType_t klass )
 {
-// Always TRUE for now (pre-empts certain optimizations).
-  return kTRUE;
+   int is_complex = 1;
+   size_t nbases = 0;
+
+   TClassRef& cr = type_from_handle( klass );
+   if ( cr.GetClass() && cr->GetListOfBases() != 0 )
+      nbases = GetNumBases( klass );
+
+   if (1 < nbases)
+      is_complex = 1;
+   else if (nbases == 0)
+      is_complex = 0;
+   else {         // one base class only
+      TBaseClass* base = (TBaseClass*)cr->GetListOfBases()->At( 0 );
+      if ( base->Property() & kIsVirtualBase )
+         is_complex = 1;       // TODO: verify; can be complex, need not be.
+      else
+         is_complex = HasComplexHierarchy( GetScope( base->GetName() ) );
+   }
+
+   return is_complex;
 }
 
 Cppyy::TCppIndex_t Cppyy::GetNumBases( TCppType_t klass )

From pypy.commits at gmail.com  Tue Aug 23 04:00:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 23 Aug 2016 01:00:26 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Will try to fix that next
Message-ID: <57bc029a.53b81c0a.4e03a.cc7a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5681:9e7e84900247
Date: 2016-08-23 10:00 +0200
http://bitbucket.org/pypy/extradoc/changeset/9e7e84900247/

Log:	Will try to fix that next

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -11,6 +11,8 @@
   We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
   Matti's opinion on that would be great!
 
+* arigo: Newly created file descriptors are non-inheritable (PEP 446)
+
 
 Finished
 --------

From pypy.commits at gmail.com  Tue Aug 23 04:05:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 23 Aug 2016 01:05:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Reduce the diff,
 don't import cpyext in this _cffi_backend test
Message-ID: <57bc03dd.c75dc20a.8a59a.9704@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86437:990c0523d383
Date: 2016-08-23 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/990c0523d383/

Log:	Reduce the diff, don't import cpyext in this _cffi_backend test

diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -79,7 +79,7 @@
 
 
 class AppTestRecompiler:
-    spaceconfig = dict(usemodules=['_cffi_backend', 'imp', 'cpyext', 'struct'])
+    spaceconfig = dict(usemodules=['_cffi_backend', 'imp'])
 
     def setup_class(cls):
         if cls.runappdirect:
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -55,9 +55,6 @@
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
 
-    # the next line is mandatory to init cpyext
-    space.getbuiltinmodule("cpyext")
-
     from pypy.module.cpyext.api import load_extension_module
     load_extension_module(space, filename, space.str_w(w_modulename))
 

From pypy.commits at gmail.com  Tue Aug 23 05:09:09 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 23 Aug 2016 02:09:09 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57bc12b5.0117c20a.c7a53.b6ef@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86438:a39f3886c9ac
Date: 2016-08-23 09:39 +0200
http://bitbucket.org/pypy/pypy/changeset/a39f3886c9ac/

Log:	merge default

diff too long, truncating to 2000 out of 5493 lines

diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -513,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -570,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -356,6 +356,11 @@
 that a C-level traceback is usually of no help at all in PyPy.
 Debugging PyPy can be annoying.
 
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
 In more details:
 
 * First, please give the exact PyPy version, and the OS.
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -119,3 +119,34 @@
 ``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
 importantly, CFFI calls that take directly a string as argument don't
 copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -24,11 +24,15 @@
 -V     : print the Python version number and exit (also --version)
 -W arg : warning control; arg is action:message:category:module:lineno
          also PYTHONWARNINGS=arg
+-X arg : set implementation-specific option
 file   : program read from script file
 -      : program read from stdin (default; interactive mode if a tty)
 arg ...: arguments passed to program in sys.argv[1:]
+
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
+-X track-resources : track the creation of files and sockets and display
+                     a warning if they are not closed explicitly
 """
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
@@ -229,6 +233,14 @@
         import pypyjit
         pypyjit.set_param(jitparam)
 
+def set_runtime_options(options, Xparam, *args):
+    if Xparam == 'track-resources':
+        sys.pypy_set_track_resources(True)
+    else:
+        print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),)
+        print >> sys.stderr, '[options] can be: track-resources'
+        raise SystemExit
+
 class CommandLineError(Exception):
     pass
 
@@ -404,6 +416,7 @@
     '--info':    (print_info,      None),
     '--jit':     (set_jit_option,  Ellipsis),
     '-funroll-loops': (funroll_loops, None),
+    '-X':        (set_runtime_options, Ellipsis),
     '--':        (end_options,     None),
     }
 
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -387,7 +387,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -406,8 +407,11 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            assert depth >= 0
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -108,8 +108,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1156,3 +1156,22 @@
             counts = self.count_instructions(source)
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
+
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1703,6 +1703,23 @@
                 "Python int too large for C unsigned short")
         return value
 
+    def c_uid_t_w(self, w_obj):
+        # xxx assumes that uid_t and gid_t are a C unsigned int.
+        # Equivalent to space.c_uint_w(), with the exception that
+        # it also accepts -1 and converts that to UINT_MAX, which
+        # is (uid_t)-1.  And values smaller than -1 raise
+        # OverflowError, not ValueError.
+        try:
+            return self.c_uint_w(w_obj)
+        except OperationError as e:
+            if e.match(self, self.w_ValueError):
+                # ValueError: cannot convert negative integer to unsigned
+                if self.int_w(w_obj) == -1:
+                    return UINT_MAX
+                raise oefmt(self.w_OverflowError,
+                            "user/group id smaller than minimum (-1)")
+            raise
+
     def truncatedint_w(self, w_obj, allow_conversion=True):
         # Like space.gateway_int_w(), but return the integer truncated
         # instead of raising OverflowError.  For obscure cases only.
@@ -1764,6 +1781,40 @@
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
         """)
 
+    def resource_warning(self, w_msg, w_tb):
+        self.appexec([w_msg, w_tb],
+                     """(msg, tb):
+            import sys
+            print >> sys.stderr, msg
+            if tb:
+                print >> sys.stderr, "Created at (most recent call last):"
+                print >> sys.stderr, tb
+        """)
+
+    def format_traceback(self):
+        # we need to disable track_resources before calling the traceback
+        # module. Else, it tries to open more files to format the traceback,
+        # the file constructor will call space.format_traceback etc., in an
+        # inifite recursion
+        flag = self.sys.track_resources
+        self.sys.track_resources = False
+        try:
+            return self.appexec([],
+                         """():
+                import sys, traceback
+                # the "1" is because we don't want to show THIS code
+                # object in the traceback
+                try:
+                    f = sys._getframe(1)
+                except ValueError:
+                    # this happens if you call format_traceback at the very beginning
+                    # of startup, when there is no bottom code object
+                    return '<no stacktrace available>'
+                return "".join(traceback.format_stack(f))
+            """)
+        finally:
+            self.sys.track_resources = flag
+
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -167,6 +167,9 @@
     def visit_c_ushort(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_c_uid_t(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_truncatedint_w(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -294,6 +297,9 @@
     def visit_c_ushort(self, typ):
         self.run_args.append("space.c_ushort_w(%s)" % (self.scopenext(),))
 
+    def visit_c_uid_t(self, typ):
+        self.run_args.append("space.c_uid_t_w(%s)" % (self.scopenext(),))
+
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
@@ -440,6 +446,9 @@
     def visit_c_ushort(self, typ):
         self.unwrap.append("space.c_ushort_w(%s)" % (self.nextarg(),))
 
+    def visit_c_uid_t(self, typ):
+        self.unwrap.append("space.c_uid_t_w(%s)" % (self.nextarg(),))
+
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
@@ -587,7 +596,10 @@
 
         # First extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(func.func_code)
+        sig = pycode.cpython_code_signature(func.func_code)
+        argnames = sig.argnames
+        varargname = sig.varargname
+        kwargname = sig.kwargname
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -611,7 +623,9 @@
         app_sig = SignatureBuilder(func)
 
         UnwrapSpec_Check(orig_sig).apply_over(unwrap_spec, app_sig)
-        self.sig = argnames, varargname, kwargname = app_sig.signature()
+        self.sig = app_sig.signature()
+        argnames = self.sig.argnames
+        varargname = self.sig.varargname
 
         self.minargs = len(argnames)
         if varargname:
@@ -942,7 +956,7 @@
                 defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
             UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig[0])
+            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
             if defs_w:
                 alldefs_w[-len(defs_w):] = defs_w
             code = self._code
@@ -959,7 +973,7 @@
                     assert isinstance(w_default, W_Root)
                     assert argname.startswith('w_')
                     argname = argname[2:]
-                    j = self._code.sig[0].index(argname)
+                    j = self._code.sig.argnames.index(argname)
                     assert alldefs_w[j] in (UNDEFINED, None)
                     alldefs_w[j] = w_default
             first_defined = 0
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -37,7 +37,7 @@
 
 # cpython_code_signature helper
 def cpython_code_signature(code):
-    "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
+    """Return a Signature instance."""
     argcount = code.co_argcount
     varnames = code.co_varnames
     assert argcount >= 0     # annotator hint
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -55,18 +55,3 @@
         if not isinstance(other, Signature):
             return NotImplemented
         return not self == other
-
-
-    # make it look tuply for its use in the annotator
-
-    def __len__(self):
-        return 3
-
-    def __getitem__(self, i):
-        if i == 0:
-            return self.argnames
-        if i == 1:
-            return self.varargname
-        if i == 2:
-            return self.kwargname
-        raise IndexError
\ No newline at end of file
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -220,6 +220,13 @@
         expected = {"no_user_site": True}
         self.check(['-c', 'pass'], {}, sys_argv=['-c'], run_command='pass', **expected)
 
+    def test_track_resources(self, monkeypatch):
+        myflag = [False]
+        def pypy_set_track_resources(flag):
+            myflag[0] = flag
+        monkeypatch.setattr(sys, 'pypy_set_track_resources', pypy_set_track_resources, raising=False)
+        self.check(['-X', 'track-resources'], {}, sys_argv=[''], run_stdin=True)
+        assert myflag[0] == True
 
 class TestInteraction:
     """
@@ -1074,4 +1081,3 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -46,13 +46,6 @@
         assert sig.find_argname("c") == 2
         assert sig.find_argname("d") == -1
 
-    def test_tuply(self):
-        sig = Signature(["a", "b", "c"], "d", "e")
-        x, y, z = sig
-        assert x == ["a", "b", "c"]
-        assert y == "d"
-        assert z == "e"
-
 class dummy_wrapped_dict(dict):
     def __nonzero__(self):
         raise NotImplementedError
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -427,3 +427,28 @@
         space.finish()
         # assert that we reach this point without getting interrupted
         # by the OperationError(NameError)
+
+    def test_format_traceback(self):
+        from pypy.tool.pytest.objspace import maketestobjspace
+        from pypy.interpreter.gateway import interp2app
+        #
+        def format_traceback(space):
+            return space.format_traceback()
+        #
+        space = maketestobjspace()
+        w_format_traceback = space.wrap(interp2app(format_traceback))
+        w_tb = space.appexec([w_format_traceback], """(format_traceback):
+            def foo():
+                return bar()
+            def bar():
+                return format_traceback()
+            return foo()
+        """)
+        tb = space.str_w(w_tb)
+        expected = '\n'.join([
+            '  File "?", line 6, in anonymous',  # this is the appexec code object
+            '  File "?", line 3, in foo',
+            '  File "?", line 5, in bar',
+            ''
+        ])
+        assert tb == expected
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -203,7 +203,8 @@
     name = func.__name__
     extra = ', '.join(extraargs)
     from pypy.interpreter import pycode
-    argnames, _, _ = pycode.cpython_code_signature(func.func_code)
+    sig = pycode.cpython_code_signature(func.func_code)
+    argnames = sig.argnames
     if use_closure:
         if argnames[1] == 'space':
             args = "closure, space, obj"
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.7.0"
+VERSION = "1.8.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -310,11 +310,15 @@
                             self.ctype.name, ct.name)
             #
             itemsize = ct.ctitem.size
-            if itemsize <= 0:
-                itemsize = 1
             with self as ptr1, w_other as ptr2:
                 diff = (rffi.cast(lltype.Signed, ptr1) -
-                        rffi.cast(lltype.Signed, ptr2)) // itemsize
+                        rffi.cast(lltype.Signed, ptr2))
+            if itemsize > 1:
+                if diff % itemsize:
+                    raise oefmt(space.w_ValueError,
+                        "pointer subtraction: the distance between the two "
+                        "pointers is not a multiple of the item size")
+                diff //= itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -253,6 +253,7 @@
     sandboxsafe=True)
 
 # split here for JIT backends that don't support floats/longlongs/etc.
+ at jit.dont_look_inside
 def is_nonnull_longdouble(cdata):
     return _is_nonnull_longdouble(read_raw_longdouble_data(cdata))
 def is_nonnull_float(cdata, size):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -576,6 +576,19 @@
     e = py.test.raises(TypeError, "q - a")
     assert str(e.value) == "cannot subtract cdata 'short *' and cdata 'int *'"
 
+def test_ptr_sub_unaligned():
+    BInt = new_primitive_type("int")
+    BIntPtr = new_pointer_type(BInt)
+    a = cast(BIntPtr, 1240)
+    for bi in range(1430, 1438):
+        b = cast(BIntPtr, bi)
+        if ((bi - 1240) % size_of_int()) == 0:
+            assert b - a == (bi - 1240) // size_of_int()
+            assert a - b == (1240 - bi) // size_of_int()
+        else:
+            py.test.raises(ValueError, "b - a")
+            py.test.raises(ValueError, "a - b")
+
 def test_cast_primitive_from_cdata():
     p = new_primitive_type("int")
     n = cast(p, cast(p, -42))
diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -69,10 +69,13 @@
         sub_ffi.set_source('re_py_subsrc', None)
         sub_ffi.emit_python_code(str(tmpdir.join('re_py_subsrc.py')))
         #
-        space.appexec([space.wrap(str(tmpdir))], """(path):
-            import _cffi_backend     # force it to be initialized
-            import sys
-            sys.path.insert(0, path)
+        cls.w_fix_path = space.appexec([space.wrap(str(tmpdir))], """(path):
+            def fix_path(ignored=None):
+                import _cffi_backend     # force it to be initialized
+                import sys
+                if path not in sys.path:
+                    sys.path.insert(0, path)
+            return fix_path
         """)
 
     def teardown_method(self, meth):
@@ -86,17 +89,20 @@
 
 
     def test_constant_1(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
 
     def test_large_constant(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const('BIGPOS') == 420000000000
         assert ffi.integer_const('BIGNEG') == -420000000000
 
     def test_function(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.add42(-10) == 32
@@ -104,6 +110,7 @@
 
     def test_dlclose(self):
         import _cffi_backend
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         ffi.dlclose(lib)
@@ -115,17 +122,20 @@
             "library '%s' has been closed" % (self.extmod,))
 
     def test_constant_via_lib(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.FOOBAR == -42
         assert lib.FOOBAZ == -43
 
     def test_opaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.cast("struct foo_s *", 0)
         raises(TypeError, ffi.new, "struct foo_s *")
 
     def test_nonopaque_struct(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         for p in [ffi.new("struct bar_s *", [5, b"foobar"]),
                   ffi.new("bar_t *", [5, b"foobar"])]:
@@ -134,12 +144,14 @@
             assert p.a[5] == ord('r')
 
     def test_enum(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         assert ffi.integer_const("BB") == 1
         e = ffi.cast("enum foo_e", 2)
         assert ffi.string(e) == "CC"
 
     def test_include_1(self):
+        self.fix_path()
         from re_py_subsrc import ffi
         assert ffi.integer_const('FOOBAR') == -42
         assert ffi.integer_const('FOOBAZ') == -43
@@ -153,6 +165,7 @@
         assert p.a[4] == ord('a')
 
     def test_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalvar42 == 1234
@@ -163,24 +176,28 @@
         assert lib.globalvar42 == 1238
 
     def test_global_const_int(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert lib.globalconst42 == 4321
         raises(AttributeError, ffi.addressof, lib, 'globalconst42')
 
     def test_global_const_nonint(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         assert ffi.string(lib.globalconsthello, 8) == "hello"
         raises(AttributeError, ffi.addressof, lib, 'globalconsthello')
 
     def test_rtld_constants(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         ffi.RTLD_NOW    # check that we have the attributes
         ffi.RTLD_LAZY
         ffi.RTLD_GLOBAL
 
     def test_no_such_function_or_global_var(self):
+        self.fix_path()
         from re_python_pysrc import ffi
         lib = ffi.dlopen(self.extmod)
         e = raises(ffi.error, getattr, lib, 'no_such_function')
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -38,23 +38,33 @@
     errors   = None
     fd       = -1
     cffi_fileobj = None    # pypy/module/_cffi_backend
+    w_tb     = None  # String representation of the traceback at creation time
 
     newlines = 0     # Updated when the stream is closed
 
     def __init__(self, space):
         self.space = space
         self.register_finalizer(space)
+        if self.space.sys.track_resources:
+            self.w_tb = self.space.format_traceback()
 
     def _finalize_(self):
         # assume that the file and stream objects are only visible in the
         # thread that runs _finalize_, so no race condition should be
         # possible and no locking is done here.
-        if self.stream is not None:
-            try:
-                self.direct_close()
-            except StreamErrors as e:
-                operr = wrap_streamerror(self.space, e, self.w_name)
-                raise operr
+        if self.stream is None:
+            return
+        if self.space.sys.track_resources:
+            w_repr = self.space.repr(self)
+            str_repr = self.space.str_w(w_repr)
+            w_msg = self.space.wrap("WARNING: unclosed file: " + str_repr)
+            self.space.resource_warning(w_msg, self.w_tb)
+        #
+        try:
+            self.direct_close()
+        except StreamErrors as e:
+            operr = wrap_streamerror(self.space, e, self.w_name)
+            raise operr
 
     def fdopenstream(self, stream, fd, mode, w_name=None):
         self.fd = fd
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,6 @@
 from __future__ import with_statement
-import py, os, errno
+import pytest, os, errno
+from pypy.interpreter.gateway import interp2app, unwrap_spec
 
 def getfile(space):
     return space.appexec([], """():
@@ -10,13 +11,24 @@
             return file
     """)
 
+# the following function is used e.g. in test_resource_warning
+ at unwrap_spec(regex=str, s=str)
+def regex_search(space, regex, s):
+    import re
+    import textwrap
+    regex = textwrap.dedent(regex).strip()
+    m = re.search(regex, s)
+    m = bool(m)
+    return space.wrap(m)
+
 class AppTestFile(object):
     spaceconfig = dict(usemodules=("_file",))
 
     def setup_class(cls):
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+            str(pytest.ensuretemp("fileimpl").join("foo.txt")))
         cls.w_file = getfile(cls.space)
+        cls.w_regex_search = cls.space.wrap(interp2app(regex_search))
 
     def test_simple(self):
         f = self.file(self.temppath, "w")
@@ -206,6 +218,9 @@
         assert exc.value.filename == os.curdir
 
     def test_encoding_errors(self):
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            pytest.skip("pypy only test")
         import _file
 
         with self.file(self.temppath, "w") as f:
@@ -254,6 +269,71 @@
         if '__pypy__' in sys.builtin_module_names:
             assert repr(self.temppath) in g.getvalue()
 
+    @pytest.mark.skipif("config.option.runappdirect")
+    def test_track_resources(self):
+        import os, gc, sys, cStringIO
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("pypy specific test")
+        def fn(flag1, flag2, do_close=False):
+            sys.pypy_set_track_resources(flag1)
+            f = self.file(self.temppath, 'w')
+            sys.pypy_set_track_resources(flag2)
+            buf = cStringIO.StringIO()
+            preverr = sys.stderr
+            try:
+                sys.stderr = buf
+                if do_close:
+                    f.close()
+                del f
+                gc.collect() # force __del__ to be called
+            finally:
+                sys.stderr = preverr
+                sys.pypy_set_track_resources(False)
+            return buf.getvalue()
+
+        # check with track_resources disabled
+        assert fn(False, False) == ""
+        #
+        # check that we don't get the warning if we actually close the file
+        assert fn(False, False, do_close=True) == ""
+        #
+        # check with track_resources enabled
+        msg = fn(True, True)
+        assert self.regex_search(r"""
+        WARNING: unclosed file: <open file .*>
+        Created at \(most recent call last\):
+          File ".*", line .*, in test_track_resources
+          File ".*", line .*, in fn
+        """, msg)
+        #
+        # check with track_resources enabled in the destructor BUT with a
+        # file which was created when track_resources was disabled
+        msg = fn(False, True)
+        assert self.regex_search("WARNING: unclosed file: <open file .*>", msg)
+        assert "Created at" not in msg
+
+    @pytest.mark.skipif("config.option.runappdirect")
+    def test_track_resources_dont_crash(self):
+        import os, gc, sys, cStringIO
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("pypy specific test")
+        #
+        # try hard to create a code object whose co_filename points to an
+        # EXISTING file, so that traceback.py tries to open it when formatting
+        # the stacktrace
+        f = open(self.temppath, 'w')
+        f.close()
+        co = compile('open("%s")' % self.temppath, self.temppath, 'exec')
+        sys.pypy_set_track_resources(True)
+        try:
+            # this exec used to fail, because space.format_traceback tried to
+            # recurively open a file, causing an infinite recursion. For the
+            # purpose of this test, it is enough that it actually finishes
+            # without errors
+            exec co
+        finally:
+            sys.pypy_set_track_resources(False)
+
     def test_truncate(self):
         f = self.file(self.temppath, "w")
         f.write("foo")
@@ -313,7 +393,7 @@
         cls.old_read = os.read
 
         if cls.runappdirect:
-            py.test.skip("works with internals of _file impl on py.py")
+            pytest.skip("works with internals of _file impl on py.py")
         def read(fd, n=None):
             if fd != 424242:
                 return cls.old_read(fd, n)
@@ -352,9 +432,9 @@
 
     def setup_class(cls):
         if not cls.runappdirect:
-            py.test.skip("likely to deadlock when interpreted by py.py")
+            pytest.skip("likely to deadlock when interpreted by py.py")
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("concurrency.txt")))
+            str(pytest.ensuretemp("fileimpl").join("concurrency.txt")))
         cls.w_file = getfile(cls.space)
 
     def test_concurrent_writes(self):
@@ -465,7 +545,7 @@
 
     def setup_class(cls):
         cls.w_temppath = cls.space.wrap(
-            str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+            str(pytest.ensuretemp("fileimpl").join("foo.txt")))
         cls.w_file = getfile(cls.space)
 
     def test___enter__(self):
diff --git a/pypy/module/_jitlog/test/test__jitlog.py b/pypy/module/_jitlog/test/test__jitlog.py
--- a/pypy/module/_jitlog/test/test__jitlog.py
+++ b/pypy/module/_jitlog/test/test__jitlog.py
@@ -10,10 +10,10 @@
 
     def setup_class(cls):
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__jitlog.1')))
-        cls.w_mark_header = cls.space.wrap(jl.MARK_JITLOG_HEADER)
-        cls.w_version = cls.space.wrap(jl.JITLOG_VERSION_16BIT_LE)
+        cls.w_mark_header = cls.space.newbytes(jl.MARK_JITLOG_HEADER)
+        cls.w_version = cls.space.newbytes(jl.JITLOG_VERSION_16BIT_LE)
         cls.w_is_32bit = cls.space.wrap(sys.maxint == 2**31-1)
-        cls.w_machine = cls.space.wrap(platform.machine())
+        cls.w_machine = cls.space.newbytes(platform.machine())
         cls.w_resops = cls.space.newdict()
         space = cls.space
         for key, value in opname.items():
@@ -48,5 +48,3 @@
                 assert opnum in self.resops
                 # the name must equal
                 assert self.resops[opnum] == opname
-
-
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h
@@ -268,22 +268,26 @@
     min = 0;
     max = haystacksize;
 
-    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
+    for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
         if (value < haystack[pos].uniseq) {
-            if (max == pos) break;
-            else max = pos;
+            if (max != pos) {
+                max = pos;
+                continue;
+            }
         }
         else if (value > haystack[pos].uniseq) {
-            if (min == pos) break;
-            else min = pos;
+            if (min != pos) {
+                min = pos;
+                continue;
+            }
         }
-        else
-            break;
+        break;
+    }
 
-        if (value == haystack[pos].uniseq)
-            return haystack[pos].code;
-        else
-            return DBCINV;
+    if (value == haystack[pos].uniseq) {
+        return haystack[pos].code;
+    }
+    return DBCINV;
 }
 #endif
 
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -151,9 +151,23 @@
 
 
 class W_Socket(W_Root):
+    w_tb = None  # String representation of the traceback at creation time
+
     def __init__(self, space, sock):
+        self.space = space
         self.sock = sock
         register_socket(space, sock)
+        if self.space.sys.track_resources:
+            self.w_tb = self.space.format_traceback()
+            self.register_finalizer(space)
+
+    def _finalize_(self):
+        is_open = self.sock.fd >= 0
+        if is_open and self.space.sys.track_resources:
+            w_repr = self.space.repr(self)
+            str_repr = self.space.str_w(w_repr)
+            w_msg = self.space.wrap("WARNING: unclosed " + str_repr)
+            self.space.resource_warning(w_msg, self.w_tb)
 
     def get_type_w(self, space):
         return space.wrap(self.sock.type)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,6 +1,8 @@
 import sys, os
-import py
+import pytest
 from pypy.tool.pytest.objspace import gettestobjspace
+from pypy.interpreter.gateway import interp2app
+from pypy.module._file.test.test_file import regex_search
 from rpython.tool.udir import udir
 from rpython.rlib import rsocket
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -12,8 +14,6 @@
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
     mod.path = udir.join('fd')
     mod.path.write('fo')
-    mod.raises = py.test.raises # make raises available from app-level tests
-    mod.skip = py.test.skip
 
 def test_gethostname():
     host = space.appexec([w_socket], "(_socket): return _socket.gethostname()")
@@ -41,7 +41,7 @@
     for host in ["localhost", "127.0.0.1", "::1"]:
         if host == "::1" and not ipv6:
             from pypy.interpreter.error import OperationError
-            with py.test.raises(OperationError):
+            with pytest.raises(OperationError):
                 space.appexec([w_socket, space.wrap(host)],
                               "(_socket, host): return _socket.gethostbyaddr(host)")
             continue
@@ -57,14 +57,14 @@
     assert space.unwrap(port) == 25
     # 1 arg version
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyname second argument is not optional before python 2.4")
+        pytest.skip("getservbyname second argument is not optional before python 2.4")
     port = space.appexec([w_socket, space.wrap(name)],
                         "(_socket, name): return _socket.getservbyname(name)")
     assert space.unwrap(port) == 25
 
 def test_getservbyport():
     if sys.version_info < (2, 4):
-        py.test.skip("getservbyport does not exist before python 2.4")
+        pytest.skip("getservbyport does not exist before python 2.4")
     port = 25
     # 2 args version
     name = space.appexec([w_socket, space.wrap(port)],
@@ -97,7 +97,7 @@
 def test_fromfd():
     # XXX review
     if not hasattr(socket, 'fromfd'):
-        py.test.skip("No socket.fromfd on this platform")
+        pytest.skip("No socket.fromfd on this platform")
     orig_fd = path.open()
     fd = space.appexec([w_socket, space.wrap(orig_fd.fileno()),
             space.wrap(socket.AF_INET), space.wrap(socket.SOCK_STREAM),
@@ -157,7 +157,7 @@
 
 def test_pton_ntop_ipv4():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     tests = [
         ("123.45.67.89", "\x7b\x2d\x43\x59"),
         ("0.0.0.0", "\x00" * 4),
@@ -173,9 +173,9 @@
 
 def test_ntop_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -194,9 +194,9 @@
 
 def test_pton_ipv6():
     if not hasattr(socket, 'inet_pton'):
-        py.test.skip('No socket.inet_pton on this platform')
+        pytest.skip('No socket.inet_pton on this platform')
     if not socket.has_ipv6:
-        py.test.skip("No IPv6 on this platform")
+        pytest.skip("No IPv6 on this platform")
     tests = [
         ("\x00" * 16, "::"),
         ("\x01" * 16, ":".join(["101"] * 8)),
@@ -215,7 +215,7 @@
         assert space.unwrap(w_packed) == packed
 
 def test_has_ipv6():
-    py.test.skip("has_ipv6 is always True on PyPy for now")
+    pytest.skip("has_ipv6 is always True on PyPy for now")
     res = space.appexec([w_socket], "(_socket): return _socket.has_ipv6")
     assert space.unwrap(res) == socket.has_ipv6
 
@@ -229,7 +229,7 @@
     w_l = space.appexec([w_socket, space.wrap(host), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, long(port))")
     assert space.unwrap(w_l) == info
-    py.test.skip("Unicode conversion is too slow")
+    pytest.skip("Unicode conversion is too slow")
     w_l = space.appexec([w_socket, space.wrap(unicode(host)), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, port)")
     assert space.unwrap(w_l) == info
@@ -250,7 +250,7 @@
 def test_addr_raw_packet():
     from pypy.module._socket.interp_socket import addr_as_object
     if not hasattr(rsocket._c, 'sockaddr_ll'):
-        py.test.skip("posix specific test")
+        pytest.skip("posix specific test")
     # HACK: To get the correct interface number of lo, which in most cases is 1,
     # but can be anything (i.e. 39), we need to call the libc function
     # if_nametoindex to get the correct index
@@ -314,6 +314,7 @@
     def setup_class(cls):
         cls.space = space
         cls.w_udir = space.wrap(str(udir))
+        cls.w_regex_search = space.wrap(interp2app(regex_search))
 
     def teardown_class(cls):
         if not cls.runappdirect:
@@ -402,6 +403,64 @@
         if os.name != 'nt':
             raises(OSError, os.close, fileno)
 
+    def test_socket_track_resources(self):
+        import _socket, os, gc, sys, cStringIO
+        s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+        fileno = s.fileno()
+        assert s.fileno() >= 0
+        s.close()
+        assert s.fileno() < 0
+        s.close()
+        if os.name != 'nt':
+            raises(OSError, os.close, fileno)
+
+    @pytest.mark.skipif("config.option.runappdirect")
+    def test_track_resources(self):
+        import os, gc, sys, cStringIO
+        import _socket
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("pypy specific test")
+        #
+        def fn(flag1, flag2, do_close=False):
+            sys.pypy_set_track_resources(flag1)
+            mysock = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+            sys.pypy_set_track_resources(flag2)
+            buf = cStringIO.StringIO()
+            preverr = sys.stderr
+            try:
+                sys.stderr = buf
+                if do_close:
+                    mysock.close()
+                del mysock
+                gc.collect() # force __del__ to be called
+            finally:
+                sys.stderr = preverr
+                sys.pypy_set_track_resources(False)
+            return buf.getvalue()
+
+        # check with track_resources disabled
+        assert fn(False, False) == ""
+        #
+        # check that we don't get the warning if we actually closed the socket
+        msg = fn(True, True, do_close=True)
+        assert msg == ''
+        #
+        # check with track_resources enabled
+        msg = fn(True, True)
+        assert self.regex_search(r"""
+        WARNING: unclosed <socket object, .*>
+        Created at \(most recent call last\):
+          File ".*", line .*, in test_track_resources
+          File ".*", line .*, in fn
+        """, msg)
+        #
+        # track_resources is enabled after the construction of the socket. in
+        # this case, the socket is not registered for finalization at all, so
+        # we don't see a message
+        msg = fn(False, True)
+        assert msg == ''
+
+
     def test_socket_close_error(self):
         import _socket, os
         if os.name == 'nt':
@@ -630,11 +689,11 @@
 class AppTestNetlink:
     def setup_class(cls):
         if not hasattr(os, 'getpid'):
-            py.test.skip("AF_NETLINK needs os.getpid()")
+            pytest.skip("AF_NETLINK needs os.getpid()")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_NETLINK')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_NETLINK on this platform")
+            pytest.skip("no AF_NETLINK on this platform")
         cls.space = space
 
     def test_connect_to_kernel_netlink_routing_socket(self):
@@ -650,11 +709,11 @@
 class AppTestPacket:
     def setup_class(cls):
         if not hasattr(os, 'getuid') or os.getuid() != 0:
-            py.test.skip("AF_PACKET needs to be root for testing")
+            pytest.skip("AF_PACKET needs to be root for testing")
         w_ok = space.appexec([], "(): import _socket; " +
                                  "return hasattr(_socket, 'AF_PACKET')")
         if not space.is_true(w_ok):
-            py.test.skip("no AF_PACKET on this platform")
+            pytest.skip("no AF_PACKET on this platform")
         cls.space = space
 
     def test_convert_between_tuple_and_sockaddr_ll(self):
diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -175,7 +175,7 @@
          "__new__"      : make_new(class_name),
          }
     pycppclass = metacpp(class_name, _drop_cycles(bases), d)
- 
+
     # cache result early so that the class methods can find the class itself
     setattr(scope, final_class_name, pycppclass)
 
@@ -192,13 +192,10 @@
     for dm_name in cppclass.get_datamember_names():
         cppdm = cppclass.get_datamember(dm_name)
 
-        # here, setattr() can not be used, because a data member can shadow one in
-        # its base class, resulting in the __set__() of its base class being called
-        # by setattr(); so, store directly on the dictionary
-        pycppclass.__dict__[dm_name] = cppdm
+        setattr(pycppclass, dm_name, cppdm)
         import cppyy
         if cppyy._is_static(cppdm):     # TODO: make this a method of cppdm
-            metacpp.__dict__[dm_name] = cppdm
+            setattr(metacpp, dm_name, cppdm)
 
     # the call to register will add back-end specific pythonizations and thus
     # needs to run first, so that the generic pythonizations can use them
@@ -413,7 +410,7 @@
         lib = cppyy._load_dictionary(name)
         _loaded_dictionaries[name] = lib
         return lib
-    
+
 def _init_pythonify():
     # cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause cppyy to be loaded
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,6 +11,9 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
 from rpython.rlib.objectmodel import dont_inline
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite,
+        c_fdopen, c_fileno,
+        c_fopen)# for tests
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import NameManager
@@ -85,44 +88,32 @@
 assert CONST_WSTRING == rffi.CWCHARP
 
 # FILE* interface
-FILEP = rffi.COpaquePtr('FILE')
 
 if sys.platform == 'win32':
     dash = '_'
 else:
     dash = ''
-fileno = rffi.llexternal(dash + 'fileno', [FILEP], rffi.INT)
-fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
-fdopen = rffi.llexternal(dash + 'fdopen', [rffi.INT, CONST_STRING],
-                  FILEP, save_err=rffi.RFFI_SAVE_ERRNO)
 
-_fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
-    if not is_valid_fd(fileno(fp)):
+    if not is_valid_fd(c_fileno(fp)):
         return -1
-    return _fclose(fp)
+    return c_fclose(fp)
 
-_fwrite = rffi.llexternal('fwrite',
-                         [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                         rffi.SIZE_T)
 def fwrite(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fwrite(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fwrite(buf, sz, n, fp)
 
-_fread = rffi.llexternal('fread',
-                        [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                        rffi.SIZE_T)
 def fread(buf, sz, n, fp):
-    validate_fd(fileno(fp))
-    return _fread(buf, sz, n, fp)
+    validate_fd(c_fileno(fp))
+    return c_fread(buf, sz, n, fp)
 
 _feof = rffi.llexternal('feof', [FILEP], rffi.INT)
 def feof(fp):
-    validate_fd(fileno(fp))
+    validate_fd(c_fileno(fp))
     return _feof(fp)
 
 def is_valid_fp(fp):
-    return is_valid_fd(fileno(fp))
+    return is_valid_fd(c_fileno(fp))
 
 pypy_decl = 'pypy_decl.h'
 
@@ -268,14 +259,14 @@
 
         # extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(callable.func_code)
+        sig = pycode.cpython_code_signature(callable.func_code)
+        assert sig.argnames[0] == 'space'
+        self.argnames = sig.argnames[1:]
+        if gil == 'pygilstate_ensure':
+            assert self.argnames[-1] == 'previous_state'
+            del self.argnames[-1]
+        assert len(self.argnames) == len(self.argtypes)
 
-        assert argnames[0] == 'space'
-        if gil == 'pygilstate_ensure':
-            assert argnames[-1] == 'previous_state'
-            del argnames[-1]
-        self.argnames = argnames[1:]
-        assert len(self.argnames) == len(self.argtypes)
         self.gil = gil
         self.result_borrowed = result_borrowed
         self.result_is_ll = result_is_ll
diff --git a/pypy/module/cpyext/c-api.txt b/pypy/module/cpyext/c-api.txt
deleted file mode 100644
--- a/pypy/module/cpyext/c-api.txt
+++ /dev/null
@@ -1,71 +0,0 @@
-Reference Count
-===============
-
-XXX
-
-Borrowed References
-===================
-
-XXX
-
-PyStringObject support
-======================
-
-The problem
------------
-
-PyString_AsString() returns a (non-movable) pointer to the underlying
-buffer, whereas pypy strings are movable.  C code may temporarily
-store this address and use it, as long as it owns a reference to the
-PyObject.  There is no "release" function to specify that the pointer
-is not needed any more.
-
-Note that the pointer may be used to fill the initial value of
-string. This is valid only when the string was just allocated, and is
-not used elsewhere.
-
-Proposed solution
------------------
-
-Our emulation of the PyStringObject contains an additional member: a
-pointer to a char buffer; it may be NULL.
-
-- A string allocated by pypy will be converted into a PyStringObject
-  with a NULL buffer.  When PyString_AsString() is called, memory is
-  allocated (with flavor='raw') and content is copied.
-
-- A string allocated with PyString_FromStringAndSize(NULL, size) will
-  allocate a buffer with the specified size, but the reference won't
-  be stored in the global map py_objects_r2w; there won't be a
-  corresponding object in pypy.  When from_ref() or Py_INCREF() is
-  called, the pypy string is created, and added in py_objects_r2w.
-  The buffer is then supposed to be immutable.
-
-- _PyString_Resize works only on not-yet-pypy'd strings, and returns a
-  similar object.
-
-- PyString_Size don't need to force the object. (in this case, another
-  "size" member is needed)
-
-- There could be an (expensive!) check in from_ref() that the buffer
-  still corresponds to the pypy gc-managed string.
-
-PySequence_Fast support
-======================
-There are five functions for fast sequence access offered by the CPython API:
-
-PyObject* PySequence_Fast(PyObject *o, const char *m)
-
-PyObject* PySequence_Fast_GET_ITEM(	PyObject *o, int i)
-
-PyObject** PySequence_Fast_ITEMS(	PyObject *o)
-
-PyObject* PySequence_ITEM(	PyObject *o, int i)
-
-int PySequence_Fast_GET_SIZE(	PyObject *o)
-
-PyPy supports four of these, but does not support PySequence_Fast_ITEMS.
-(Various ways to support PySequence_Fast_ITEMS were considered. They all had
-two things in common: they would have taken a lot of work, and they would have
-resulted in incomplete semantics or in poor performance. We decided that a slow
-implementation of PySequence_Fast_ITEMS was not very useful.)
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -2,6 +2,9 @@
 #define Py_PYTHON_H
 
 /* Compat stuff */
+#ifdef __GNUC__
+#define _GNU_SOURCE 1
+#endif
 #ifndef _WIN32
 # include <inttypes.h>
 # include <stdint.h>
@@ -52,7 +55,6 @@
 #ifndef DL_IMPORT
 #       define DL_IMPORT(RTYPE) RTYPE
 #endif
-
 #include <stdlib.h>
 
 #ifndef _WIN32
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -21,6 +21,8 @@
                          flavor='raw',
                          add_memory_pressure=True)
 
+realloc = rffi.llexternal('realloc', [rffi.VOIDP, rffi.SIZE_T], rffi.VOIDP)
+
 @cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
@@ -28,7 +30,7 @@
                          flavor='raw',
                          add_memory_pressure=True)
     # XXX FIXME
-    return lltype.nullptr(rffi.VOIDP.TO)
+    return realloc(ptr, size)
 
 @cpython_api([rffi.VOIDP], lltype.Void)
 def PyObject_Free(space, ptr):
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -1,6 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rfile import c_setvbuf, _IONBF
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers, fdopen)
+    cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers, c_fdopen)
 from pypy.module.cpyext.pyobject import PyObject
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from pypy.interpreter.error import (OperationError, oefmt, 
@@ -64,11 +65,12 @@
     if (fd < 0 or not mode or mode[0] not in ['r', 'w', 'a', 'U'] or
         ('U' in mode and ('w' in mode or 'a' in mode))):
         raise oefmt(space.w_IOError, 'invalid fileno or mode') 
-    ret = fdopen(fd, mode)
+    ret = c_fdopen(fd, mode)
     if not ret:
         raise exception_from_saved_errno(space, space.w_IOError)
+    # XXX fix this once use-file-star-for-file lands
+    c_setvbuf(ret, lltype.nullptr(rffi.CCHARP.TO), _IONBF, 0)
     return ret
-        
 
 @cpython_api([FILEP, CONST_STRING, CONST_STRING, rffi.VOIDP], PyObject)
 def PyFile_FromFile(space, fp, name, mode, close):
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -10,7 +10,7 @@
 from pypy.objspace.std import tupleobject
 
 from pypy.module.cpyext.tupleobject import PyTuple_Check, PyTuple_SetItem
-from pypy.module.cpyext.object import Py_IncRef, Py_DecRef
+from pypy.module.cpyext.pyobject import decref
 
 from pypy.module.cpyext.dictobject import PyDict_Check
 
@@ -252,7 +252,7 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        Py_DecRef(w_list.space, storage._elems[index])
+        decref(w_list.space, storage._elems[index])
         storage._elems[index] = make_ref(w_list.space, w_obj)
 
     def length(self, w_list):
@@ -264,9 +264,8 @@
         return storage._elems
 
     def getslice(self, w_list, start, stop, step, length):
-        #storage = self.unerase(w_list.lstorage)
-        raise oefmt(w_list.space.w_NotImplementedError,
-                    "settting a slice of a PySequence_Fast is not supported")
+        w_list.switch_to_object_strategy()
+        return w_list.strategy.getslice(w_list, start, stop, step, length)
 
     def getitems(self, w_list):
         # called when switching list strategy, so convert storage
@@ -389,5 +388,5 @@
 
     def __del__(self):
         for i in range(self._length):
-            Py_DecRef(self.space, self._elems[i])
+            decref(self.space, self._elems[i])
         lltype.free(self._elems, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.eval import (
     Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP
+from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
@@ -130,19 +130,19 @@
     def test_run_file(self, space, api):
         filepath = udir / "cpyext_test_runfile.py"
         filepath.write("raise ZeroDivisionError")
-        fp = fopen(str(filepath), "rb")
+        fp = c_fopen(str(filepath), "rb")
         filename = rffi.str2charp(str(filepath))
         w_globals = w_locals = space.newdict()
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_ZeroDivisionError
         api.PyErr_Clear()
 
         # try again, but with a closed file
-        fp = fopen(str(filepath), "rb")
-        os.close(fileno(fp))
+        fp = c_fopen(str(filepath), "rb")
+        os.close(c_fileno(fp))
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        fclose(fp)
+        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -235,8 +235,9 @@
         assert type(x) is int
         assert x == -424344
 
-    @pytest.mark.skipif(True, reason='realloc not fully implemented')
     def test_object_realloc(self):
+        if not self.runappdirect:
+            skip('no untranslated support for realloc')
         module = self.import_extension('foo', [
             ("realloctest", "METH_NOARGS",
              """
@@ -244,12 +245,11 @@
                  char *copy, *orig = PyObject_MALLOC(12);
                  memcpy(orig, "hello world", 12);
                  copy = PyObject_REALLOC(orig, 15);
+                 /* realloc() takes care of freeing orig, if changed */
                  if (copy == NULL)
                      Py_RETURN_NONE;
                  ret = PyString_FromStringAndSize(copy, 12);
-                 if (copy != orig)
-                     PyObject_Free(copy);
-                 PyObject_Free(orig);
+                 PyObject_Free(copy);
                  return ret;
              """)])
         x = module.realloctest()
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,4 @@
 from pypy.conftest import option
-from pypy.module.cpyext.api import fopen, fclose, fwrite
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
@@ -133,6 +132,15 @@
                     return PyLong_FromLong(0);
                 return PyLong_FromLong(ftell(fp));
              """),
+            ("read_10", "METH_O",
+             """
+                char s[10];
+                FILE * fp = PyFile_AsFile(args);
+                if (fp == NULL)
+                    return PyLong_FromLong(0);
+                fread(s, 1, 10, fp);
+                return PyLong_FromLong(ftell(fp));
+             """),
             ])
         filename = self.udir + "/_test_file"
         with open(filename, 'w') as fid:
@@ -142,5 +150,12 @@
             t_py = fid.tell()
             assert t_py == 80
             t_c = module.get_c_tell(fid)
-        assert t_c == t_py
+            assert t_c == t_py
+            print '-------- tell ',t_c
+            t_c = module.read_10(fid)
+            assert t_c == t_py + 10
+            print '-------- tell ',t_c
+            t_py = fid.tell()
+            assert t_c == t_py, 'after a fread, c level ftell(fp) %d but PyFile.tell() %d' % (t_c, t_py)
 
+
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -78,6 +78,17 @@
         assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
         assert space.eq_w(w_t, space.wrap([1, 3, 5]))
 
+    def test_get_slice_fast(self, space, api):
+        w_t = space.wrap([1, 2, 3, 4, 5])
+        api.PySequence_Fast(w_t, "foo") # converts
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4]
+        assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4]
+
+        assert api.PySequence_DelSlice(w_t, 1, 4) == 0
+        assert space.eq_w(w_t, space.wrap([1, 5]))
+        assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0
+        assert space.eq_w(w_t, space.wrap([1, 3, 5]))
+
     def test_iter(self, space, api):
         w_t = space.wrap((1, 2))
         w_iter = api.PySeqIter_New(w_t)
@@ -226,18 +237,33 @@
         assert space.int_w(space.len(w_l)) == 10
 
 
-class XAppTestSequenceObject(AppTestCpythonExtensionBase):
-    def test_sequenceobject(self):
+class AppTestSequenceObject(AppTestCpythonExtensionBase):
+    def test_fast(self):
         module = self.import_extension('foo', [
             ("test_fast_sequence", "METH_VARARGS",
              """
-                PyObject * o = PyTuple_GetItem(args, 0);
+                int size, i;
+                PyTypeObject * common_type;
+                PyObject *foo, **objects;
+                PyObject * seq = PyTuple_GetItem(args, 0);
                 /* XXX assert it is a tuple */
-                PyObject *foo = PySequence_Fast(o, "some string");
-                PyObject ** res = PySequence_Fast_ITEMS(foo);
-                /* XXX do some kind of test on res */
-                /* XXX now what? who manages res's refcount? */
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "some string");
+                objects = PySequence_Fast_ITEMS(foo);
+                size = PySequence_Fast_GET_SIZE(seq);
+                common_type = size > 0 ? Py_TYPE(objects[0]) : NULL;
+                for (i = 1; i < size; ++i) {
+                    if (Py_TYPE(objects[i]) != common_type) {
+                        common_type = NULL;
+                        break;
+                    }
+                }
+                Py_DECREF(foo);
+                Py_DECREF(common_type);
                 return PyBool_FromLong(1);
              """)])
-        assert module.test_fast_sequence([1, 2, 3, 4])
+        s = [1, 2, 3, 4]
+        assert module.test_fast_sequence(s[0:-1])
+        assert module.test_fast_sequence(s[::-1])
 
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -282,11 +282,41 @@
                      args->ob_type->tp_dict, "copy");
                  Py_INCREF(method);
                  return method;
+             '''),
+            ("get_type_dict", "METH_O",
              '''
-             )
+                PyObject* value = args->ob_type->tp_dict;
+                if (value == NULL) value = Py_None;
+                Py_INCREF(value);
+                return value;
+             '''),
             ])
         obj = foo.new()
         assert module.read_tp_dict(obj) == foo.fooType.copy
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
+        class A(object):
+            pass
+        obj = A()
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
+        del d["_some_attribute"]
+
+        d = module.get_type_dict(1)
+        assert type(d) is dict
+        try:
+            d["_some_attribute"] = 1
+        except TypeError:  # on PyPy, int.__dict__ is really immutable
+            pass
+        else:
+            assert int._some_attribute == 1
+            del d["_some_attribute"]
 
     def test_custom_allocation(self):
         foo = self.import_module("foo")
@@ -355,6 +385,21 @@
 
         api.Py_DecRef(ref)
 
+    def test_type_dict(self, space, api):
+        w_class = space.appexec([], """():
+            class A(object):
+                pass
+            return A
+            """)
+        ref = make_ref(space, w_class)
+
+        py_type = rffi.cast(PyTypeObjectPtr, ref)
+        w_dict = from_ref(space, py_type.c_tp_dict)
+        w_name = space.wrap('a')
+        space.setitem(w_dict, w_name, space.wrap(1))
+        assert space.int_w(space.getattr(w_class, w_name)) == 1
+        space.delitem(w_dict, w_name)
+
     def test_multiple_inheritance(self, space, api):
         w_class = space.appexec([], """():
             class A(object):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -272,12 +272,12 @@
         if len(slot_names) == 1:
             if not getattr(pto, slot_names[0]):
                 setattr(pto, slot_names[0], slot_func_helper)
-        elif (w_type.getname(space) in ('list', 'tuple') and 
+        elif (w_type.getname(space) in ('list', 'tuple') and
               slot_names[0] == 'c_tp_as_number'):
             # XXX hack - hwo can we generalize this? The problem is method
             # names like __mul__ map to more than one slot, and we have no
             # convenient way to indicate which slots CPython have filled
-            # 
+            #
             # We need at least this special case since Numpy checks that
             # (list, tuple) do __not__ fill tp_as_number
             pass
@@ -860,8 +860,8 @@
 

From pypy.commits at gmail.com  Tue Aug 23 05:09:11 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 23 Aug 2016 02:09:11 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: wrong offset loaded wrong value
 (passing unpack for i64)
Message-ID: <57bc12b7.2a0dc20a.a9d54.b3c2@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86439:01d2126c6305
Date: 2016-08-23 11:08 +0200
http://bitbucket.org/pypy/pypy/changeset/01d2126c6305/

Log:	wrong offset loaded wrong value (passing unpack for i64)

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -470,8 +470,6 @@
                 self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
                 self.mc.stvx(vector, r.SCRATCH2.value, r.SP.value)
                 idx = residx
-                if not IS_BIG_ENDIAN:
-                    idx = 1 - idx
                 self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx)
                 self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
             else:
@@ -499,7 +497,7 @@
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             off = off + size * idx
             if size == 8:
-                self.mc.load(res, r.SP.value, off+size*idx)
+                self.mc.load(res, r.SP.value, off)
                 return
             elif size == 4:
                 self.mc.lwa(res, r.SP.value, off)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -789,7 +789,7 @@
         #
         looptoken = JitCellToken()
         cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
-        import pdb; pdb.set_trace()
+        #import pdb; pdb.set_trace()
         deadframe = cpu.execute_token(looptoken, *args_values)
         print(source)
         if float:
@@ -805,10 +805,10 @@
                                "[2xf64]", {'x': (50.33,4321.0)}) == 4321.0
     def test_unpack_i64(self):
         # int64
+        assert self.run_unpack("i{i} = vec_unpack_i({x}, 1, 1)",
+                               "[2xi64]", {'x': (14,15)}, float=False) == 15
         assert self.run_unpack("i{i} = vec_unpack_i({x}, 0, 1)",
                                "[2xi64]", {'x': (11,12)}, float=False) == 11
-        assert self.run_unpack("i{i} = vec_unpack_i({x}, 1, 1)",
-                               "[2xi64]", {'x': (14,15)}, float=False) == 15
 
     def test_unpack_i(self):
         for i in range(16):

From pypy.commits at gmail.com  Tue Aug 23 05:28:04 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 23 Aug 2016 02:28:04 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: implement unpacking for
 integers 8, 16, 32 bits if count == 1
Message-ID: <57bc1724.0117c20a.c7a53.bf41@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86440:eccd7c7028fc
Date: 2016-08-23 11:27 +0200
http://bitbucket.org/pypy/pypy/changeset/eccd7c7028fc/

Log:	implement unpacking for integers 8,16,32 bits if count == 1

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -465,21 +465,25 @@
         vector = vloc.value
         src = sourceloc.value
         size = op.bytesize
+        assert resultloc.is_vector_reg() # vector <- reg
+        self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
+        self.mc.stvx(vector, r.SCRATCH2.value, r.SP.value)
         if size == 8:
-            if resultloc.is_vector_reg(): # vector <- reg
-                self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
-                self.mc.stvx(vector, r.SCRATCH2.value, r.SP.value)
-                idx = residx
-                self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx)
-                self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
-            else:
-                not_implemented("64 bit float")
+            idx = residx
+            self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx)
         elif size == 4:
-            not_implemented("vec_pack_i")
+            for j in range(count):
+                i = j + residx
+                self.mc.stw(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+4*i)
         elif size == 2:
-            not_implemented("vec_pack_i")
+            for j in range(count):
+                i = j + residx
+                self.mc.sth(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+2*i)
         elif size == 1:
-            not_implemented("vec_pack_i")
+            for j in range(count):
+                i = j + residx
+                self.mc.stb(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+i)
+        self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
     def emit_vec_unpack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
@@ -507,7 +511,7 @@
                 return
             elif size == 1:
                 self.mc.lbz(res, r.SP.value, off)
-                self.mc.extsb(res, res)
+                #self.mc.extsb(res, res)
                 return
 
         not_implemented("%d bit integer, count %d" % \
@@ -715,7 +719,7 @@
         return [resloc, vloc, srcloc, imm(residx), imm(srcidx), imm(count.value)]
 
     def prepare_vec_pack_f(self, op):
-        # new_res = vec_pack_i(res, src, index, count)
+        # new_res = vec_pack_f(res, src, index, count)
         assert isinstance(op, VectorOp)
         arg = op.getarg(1)
         index = op.getarg(2)

From pypy.commits at gmail.com  Tue Aug 23 06:14:31 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 23 Aug 2016 03:14:31 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Hack around dereferencing (do it
 with custom llop)
Message-ID: <57bc2207.c4ebc20a.62fc2.cfff@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86441:fd79eaeb9674
Date: 2016-08-23 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/fd79eaeb9674/

Log:	Hack around dereferencing (do it with custom llop)

diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -1,5 +1,6 @@
 from rpython.rtyper.llannotation import SomePtr, SomeAddress, s_None
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.memory.gctransform.framework import (BaseFrameworkGCTransformer, BaseRootWalker)
 
 VISIT_FPTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
@@ -22,7 +23,8 @@
         #     s_gcref], s_gcref)
         #
         def invokecallback(root, visit_fn):
-            visit_fn(root) # FIXME: Dereference root before invoking visit_fn
+            obj = llop.qcgc_dereference(llmemory.Address, root)
+            visit_fn(obj)
         def pypy_trace_cb(obj, visit_fn):
             gc.trace(obj, invokecallback, visit_fn)
         pypy_trace_cb.c_name = "pypy_trace_cb"
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -517,6 +517,7 @@
     # __________ qcgc operations __________
     'qcgc_allocate':    LLOp(canmallocgc=True),
     'qcgc_collect':     LLOp(), # XXX: No allocations, so no canmallocgc ?
+    'qcgc_dereference': LLOp(),
 
     # __________ weakrefs __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -959,3 +959,8 @@
 
     def OP_QCGC_COLLECT(self, op):
         return 'qcgc_collect();'
+
+    def OP_QCGC_DEREFERENCE(self, op):
+        arg = self.expr(op.args[0])
+        result = self.expr(op.result)
+        return '%s = *((object_t **) %s);' % (result, arg)

From pypy.commits at gmail.com  Tue Aug 23 06:15:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 23 Aug 2016 03:15:59 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: pass unpack several on ppc
Message-ID: <57bc225f.0cce1c0a.dc8d9.faba@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86442:05e0d7bb3077
Date: 2016-08-23 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/05e0d7bb3077/

Log:	pass unpack several on ppc

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -513,6 +513,32 @@
                 self.mc.lbz(res, r.SP.value, off)
                 #self.mc.extsb(res, res)
                 return
+        else:
+            # count is not 1, but only 2 is supported for i32
+            # 4 for i16 and 8 for i8.
+            src = srcloc.value | 0b100000
+            res = resloc.value | 0b100000
+            residx = 0
+            #assert idx == 0 or idx == 8
+            if (size == 4 and count == 2) or \
+               (size == 2 and count == 4) or \
+               (size == 1 and count == 8):
+                if idx == 0:
+                    self.mc.xxpermdi(res, src, res, permi(0,1))
+                else:
+                    self.mc.xxpermdi(res, src, res, permi(1,1))
+                return
+
+            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
+            self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
+            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
+            self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
+            if count * size == 8:
+                off = PARAM_SAVE_AREA_OFFSET + idx * size
+                self.mc.load(r.SCRATCH.value, r.SP.value, off)
+                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16)
+                self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
+                return
 
         not_implemented("%d bit integer, count %d" % \
                        (size*8, count))
@@ -759,7 +785,10 @@
             # unpack
             srcloc = self.ensure_reg(arg0)
         size = arg.bytesize
-        resloc = self.force_allocate_reg(op)
+        if op.is_vector():
+            resloc = self.force_allocate_vector_reg(op)
+        else:
+            resloc = self.force_allocate_reg(op)
         return [resloc, srcloc, imm(index.value), imm(count.value), imm(size)]
 
     def expand_float(self, size, box):

From pypy.commits at gmail.com  Tue Aug 23 07:36:48 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 23 Aug 2016 04:36:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Change error message on unpack and
 extended unpack if there are more values expected than available
Message-ID: <57bc3550.85261c0a.721b3.2c7f@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86443:0a1d00e6444f
Date: 2016-08-23 13:36 +0200
http://bitbucket.org/pypy/pypy/changeset/0a1d00e6444f/

Log:	Change error message on unpack and extended unpack if there are more
	values expected than available

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -942,8 +942,8 @@
             idx += 1
         if idx < expected_length:
             raise oefmt(self.w_ValueError,
-                        "need more than %d value%s to unpack",
-                        idx, "" if idx == 1 else "s")
+                        "not enough values to unpack (expected %d, got %d)",
+                        expected_length, idx)
         return items
 
     def unpackiterable_unroll(self, w_iterable, expected_length):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -820,13 +820,9 @@
         itemcount = len(items)
         count = left + right
         if count > itemcount:
-            if count == 1:
-                plural = ''
-            else:
-                plural = 's'
             raise oefmt(self.space.w_ValueError,
-                        "need more than %d value%s to unpack",
-                        itemcount, plural)
+                        "not enough values to unpack (expected at least %d, got %d)",
+                        count, itemcount)
         right = itemcount - right
         assert right >= 0
         # push values in reverse order
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -458,7 +458,7 @@
         try:
             a, *b, c, d, e = Seq()
         except ValueError as e:
-            assert str(e) == "need more than 3 values to unpack"
+            assert str(e) == "not enough values to unpack (expected at least 4, got 3)"
         else:
             assert False, "Expected ValueError"
             """
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -401,8 +401,8 @@
                         "too many values to unpack (expected %d)", expected)
         else:
             raise oefmt(self.w_ValueError,
-                        "need more than %d value%s to unpack",
-                        got, "" if got == 1 else "s")
+                        "not enough values to unpack (expected %d, got %d)",
+                        expected, got)
 
     def unpackiterable(self, w_obj, expected_length=-1):
         if isinstance(w_obj, W_AbstractTupleObject) and self._uses_tuple_iter(w_obj):

From pypy.commits at gmail.com  Tue Aug 23 08:58:30 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 23 Aug 2016 05:58:30 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: some byte ordering issue
 remains, fixed all other vector tests on ppc
Message-ID: <57bc4876.2a0dc20a.a9d54.1410@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86444:b8fc16ac0662
Date: 2016-08-23 14:57 +0200
http://bitbucket.org/pypy/pypy/changeset/b8fc16ac0662/

Log:	some byte ordering issue remains, fixed all other vector tests on
	ppc

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -276,9 +276,9 @@
         if arg.type == FLOAT:
             # r = (r[0]+r[1],r[0]+r[1])
             if IS_BIG_ENDIAN:
-                self.mc.xxspltd(tgt, acc, acc, 0b00)
+                self.mc.xxpermdi(tgt, acc, acc, 0b00)
             else:
-                self.mc.xxspltd(tgt, acc, acc, 0b10)
+                self.mc.xxpermdi(tgt, acc, acc, 0b10)
             if op == '+':
                 self.mc.xsadddp(tgt, tgt, acc)
             elif op == '*':
@@ -468,21 +468,29 @@
         assert resultloc.is_vector_reg() # vector <- reg
         self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
         self.mc.stvx(vector, r.SCRATCH2.value, r.SP.value)
+        idx = residx
         if size == 8:
-            idx = residx
+            if not IS_BIG_ENDIAN:
+                idx = (16 // size) - 1 - idx
             self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx)
         elif size == 4:
             for j in range(count):
-                i = j + residx
-                self.mc.stw(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+4*i)
+                idx = j + residx
+                if not IS_BIG_ENDIAN:
+                    idx = (16 // size) - 1 - idx
+                self.mc.stw(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+4*idx)
         elif size == 2:
             for j in range(count):
-                i = j + residx
-                self.mc.sth(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+2*i)
+                idx = j + residx
+                if not IS_BIG_ENDIAN:
+                    idx = (16 // size) - 1 - idx
+                self.mc.sth(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+2*idx)
         elif size == 1:
             for j in range(count):
-                i = j + residx
-                self.mc.stb(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+i)
+                idx = j + residx
+                if not IS_BIG_ENDIAN:
+                    idx = (16 // size) - 1 - idx
+                self.mc.stb(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+idx)
         self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
     def emit_vec_unpack_i(self, op, arglocs, regalloc):
@@ -499,7 +507,9 @@
             off = PARAM_SAVE_AREA_OFFSET
             self.mc.load_imm(r.SCRATCH2, off)
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
-            off = off + size * idx
+            if not IS_BIG_ENDIAN:
+                idx = (16 // size) - 1 - idx
+            off += size * idx
             if size == 8:
                 self.mc.load(res, r.SP.value, off)
                 return
@@ -511,32 +521,26 @@
                 return
             elif size == 1:
                 self.mc.lbz(res, r.SP.value, off)
-                #self.mc.extsb(res, res)
+                self.mc.extsb(res, res)
                 return
         else:
             # count is not 1, but only 2 is supported for i32
             # 4 for i16 and 8 for i8.
-            src = srcloc.value | 0b100000
-            res = resloc.value | 0b100000
-            residx = 0
-            #assert idx == 0 or idx == 8
-            if (size == 4 and count == 2) or \
-               (size == 2 and count == 4) or \
-               (size == 1 and count == 8):
-                if idx == 0:
-                    self.mc.xxpermdi(res, src, res, permi(0,1))
-                else:
-                    self.mc.xxpermdi(res, src, res, permi(1,1))
-                return
+            src = srcloc.value
+            res = resloc.value
 
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
             if count * size == 8:
+                stidx = 0
+                if not IS_BIG_ENDIAN:
+                    idx = (16 // size) - 1 - idx
+                    stidx = 0
                 off = PARAM_SAVE_AREA_OFFSET + idx * size
                 self.mc.load(r.SCRATCH.value, r.SP.value, off)
-                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16)
+                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+stidx)
                 self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
                 return
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -831,21 +831,21 @@
         for i,v in enumerate(values):
             j = (i // 2) * 2
             op = ["v{v}[2xi32] = vec_unpack_i({x}, %d, 2)" % j,
-                  "i{i} = vec_unpack_i(v{v}[2xi32], %d, 1)" % i]
+                  "i{i} = vec_unpack_i(v{v}[2xi32], %d, 1)" % (i % 2)]
             assert self.run_unpack(op, "[4xi32]", {'x': values}, float=False) == v
 
         values = [1,2,3,4,5,6,7,8]
         for i,v in enumerate(values):
             j = (i // 4) * 4
             op = ["v{v}[4xi16] = vec_unpack_i({x}, %d, 4)" % j,
-                  "i{i} = vec_unpack_i(v{v}[4xi16], %d, 1)" % i]
+                  "i{i} = vec_unpack_i(v{v}[4xi16], %d, 1)" % (i % 4)]
             assert self.run_unpack(op, "[8xi16]", {'x': values}, float=False) == v
 
         values = [1,2,3,4,5,6,7,8] * 2
         for i,v in enumerate(values):
             j = (i // 8) * 8
             op = ["v{v}[8xi8] = vec_unpack_i({x}, %d, 8)" % j,
-                  "i{i} = vec_unpack_i(v{v}[8xi8], %d, 1)" % i]
+                  "i{i} = vec_unpack_i(v{v}[8xi8], %d, 1)" % (i % 8)]
             assert self.run_unpack(op, "[16xi8]", {'x': values}, float=False) == v
 
 

From pypy.commits at gmail.com  Tue Aug 23 09:14:40 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 23 Aug 2016 06:14:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix unpack_map bug saying X is not a
 mapping if another element is unpacked before (reset is_unpacking to False
 if element doesn't have to be unpacked)
Message-ID: <57bc4c40.e2efc20a.ff32c.1b17@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86445:c505dc6d5e8d
Date: 2016-08-23 15:13 +0200
http://bitbucket.org/pypy/pypy/changeset/c505dc6d5e8d/

Log:	Fix unpack_map bug saying X is not a mapping if another element is
	unpacked before (reset is_unpacking to False if element doesn't have
	to be unpacked)

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1236,6 +1236,8 @@
                 key = d.keys[i]
                 if key is None:
                     is_unpacking = True
+                else:
+                    is_unpacking = False
                 if elements == 0xFFFF or (elements and is_unpacking):
                     self.emit_op_arg(ops.BUILD_MAP, elements)
                     containers += 1

From pypy.commits at gmail.com  Tue Aug 23 09:25:41 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 23 Aug 2016 06:25:41 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: buffer builtin has no
 flags argument, flags was provided at the position of the offset
Message-ID: <57bc4ed5.6211c20a.7af0d.214b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86446:feafb2399ae8
Date: 2016-08-23 15:24 +0200
http://bitbucket.org/pypy/pypy/changeset/feafb2399ae8/

Log:	buffer builtin has no flags argument, flags was provided at the
	position of the offset

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -168,7 +168,7 @@
             return self.value
 
     def __buffer__(self, flags):
-        return buffer(self._buffer, flags)
+        return buffer(self._buffer)
 
     def _get_b_base(self):
         try:

From pypy.commits at gmail.com  Tue Aug 23 09:36:55 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 23 Aug 2016 06:36:55 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Replace dereference hack
Message-ID: <57bc5177.56421c0a.e1297.5138@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86447:227535124264
Date: 2016-08-23 15:36 +0200
http://bitbucket.org/pypy/pypy/changeset/227535124264/

Log:	Replace dereference hack

diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -23,8 +23,8 @@
         #     s_gcref], s_gcref)
         #
         def invokecallback(root, visit_fn):
-            obj = llop.qcgc_dereference(llmemory.Address, root)
-            visit_fn(obj)
+            ref = llmemory.cast_adr_to_ptr(root, rffi.VOIDPP)
+            visit_fn(llmemory.cast_ptr_to_adr(ref[0]))
         def pypy_trace_cb(obj, visit_fn):
             gc.trace(obj, invokecallback, visit_fn)
         pypy_trace_cb.c_name = "pypy_trace_cb"
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -517,7 +517,6 @@
     # __________ qcgc operations __________
     'qcgc_allocate':    LLOp(canmallocgc=True),
     'qcgc_collect':     LLOp(), # XXX: No allocations, so no canmallocgc ?
-    'qcgc_dereference': LLOp(),
 
     # __________ weakrefs __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -959,8 +959,3 @@
 
     def OP_QCGC_COLLECT(self, op):
         return 'qcgc_collect();'
-
-    def OP_QCGC_DEREFERENCE(self, op):
-        arg = self.expr(op.args[0])
-        result = self.expr(op.result)
-        return '%s = *((object_t **) %s);' % (result, arg)

From pypy.commits at gmail.com  Tue Aug 23 09:47:01 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 23 Aug 2016 06:47:01 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Fix collect llop not having
 canmallocgc=True
Message-ID: <57bc53d5.c2a5c20a.46c00.26a4@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86448:1b3b0d2c35be
Date: 2016-08-23 15:46 +0200
http://bitbucket.org/pypy/pypy/changeset/1b3b0d2c35be/

Log:	Fix collect llop not having canmallocgc=True

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -516,7 +516,7 @@
 
     # __________ qcgc operations __________
     'qcgc_allocate':    LLOp(canmallocgc=True),
-    'qcgc_collect':     LLOp(), # XXX: No allocations, so no canmallocgc ?
+    'qcgc_collect':     LLOp(canmallocgc=True),
 
     # __________ weakrefs __________
 

From pypy.commits at gmail.com  Tue Aug 23 09:50:43 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 23 Aug 2016 06:50:43 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Try to implement
 writebarrier_before_copy
Message-ID: <57bc54b3.4152c20a.199e7.2a3e@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86449:5af12240393f
Date: 2016-08-23 15:50 +0200
http://bitbucket.org/pypy/pypy/changeset/5af12240393f/

Log:	Try to implement writebarrier_before_copy

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -57,7 +57,10 @@
 
     def writebarrier_before_copy(self, source_addr, dest_addr,
                                  source_start, dest_start, length):
-        raise NotImplementedError
+        # XXX: Seems like returning false is the most conservative way to handle
+        # this. Unfortunately I don't fully understand what this is supposed to
+        # do, so I can't optimize it ATM.
+        return False
         # Possible implementation?
         #llop.gc_writebarrier(dest_addr)
         #return True

From pypy.commits at gmail.com  Tue Aug 23 11:13:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 23 Aug 2016 08:13:24 -0700 (PDT)
Subject: [pypy-commit] pypy redirect-assembler-jitlog: merge default
Message-ID: <57bc6814.a717c20a.2f7c9.50c2@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: redirect-assembler-jitlog
Changeset: r86450:8c7971f278b0
Date: 2016-08-23 17:12 +0200
http://bitbucket.org/pypy/pypy/changeset/8c7971f278b0/

Log:	merge default

diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -144,3 +144,9 @@
 ``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
 Previously it returned what looked like a regular dict object (but it
 was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -590,6 +590,8 @@
                                                     "decode", w_fs_encoding)
                 except OperationError as e:
                     # fall back to the original byte string
+                    if e.async(space):
+                        raise
                     result_w[i] = w_bytes
             return space.newlist(result_w)
         else:
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -423,7 +423,7 @@
         guard_false(i114, descr=...)
         --TICK--
         i123 = arraylen_gc(p67, descr=<ArrayP .>)
-        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=5 OS=110>)
+        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=<Calli . i EF=5 OS=110>)
         check_memory_error(i119)
         raw_store(i119, 0, i160, descr=<ArrayS 2>)
         raw_store(i119, 2, i160, descr=<ArrayS 2>)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1963,3 +1963,21 @@
                        ffi, "test_function_returns_opaque", "?")
     assert str(e.value) == ("function foo: 'struct a' is used as result type,"
                             " but is opaque")
+
+def test_function_returns_union():
+    ffi = FFI()
+    ffi.cdef("union u1 { int a, b; }; union u1 f1(int);")
+    lib = verify(ffi, "test_function_returns_union", """
+        union u1 { int a, b; };
+        static union u1 f1(int x) { union u1 u; u.b = x; return u; }
+    """)
+    assert lib.f1(51).a == 51
+
+def test_function_returns_partial_struct():
+    ffi = FFI()
+    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    lib = verify(ffi, "test_function_returns_partial_struct", """
+        struct a { int b, a, c; };
+        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+    """)
+    assert lib.f1(52).a == 52
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -261,6 +261,9 @@
                    "stack based virtual machines (only for backends that support it)",
                    default=True),
         BoolOption("storesink", "Perform store sinking", default=True),
+        BoolOption("replace_we_are_jitted",
+                   "Replace we_are_jitted() calls by False",
+                   default=False, cmdline=None),
         BoolOption("none",
                    "Do not run any backend optimizations",
                    requires=[('translation.backendopt.inline', False),
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -833,9 +833,6 @@
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     # vector operations
     vector_arith_code = """
     def bh_vec_{0}_{1}(self, vx, vy, count):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -757,9 +757,6 @@
             self.write_int_at_mem(res, self.vtable_offset, WORD, sizedescr.get_vtable())
         return res
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -229,8 +229,6 @@
         raise NotImplementedError
     def bh_newunicode(self, length):
         raise NotImplementedError
-    def bh_new_raw_buffer(self, size):
-        raise NotImplementedError
 
     def bh_arraylen_gc(self, array, arraydescr):
         raise NotImplementedError
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -582,6 +582,14 @@
                 return lltype.malloc(ARRAY, n, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_1_raw_malloc_varsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_1_raw_malloc_varsize.func_name = name
             return _ll_1_raw_malloc_varsize
         return build_ll_1_raw_malloc_varsize
 
@@ -610,6 +618,14 @@
                 return lltype.malloc(STRUCT, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_0_raw_malloc_fixedsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_0_raw_malloc_fixedsize.func_name = name
             return _ll_0_raw_malloc_fixedsize
         return build_ll_0_raw_malloc_fixedsize
 
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -367,8 +367,9 @@
 
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
-    
-    def __init__(self, cpu, size=-1):
+
+    def __init__(self, cpu, func, size=-1):
+        self.func = func
         self.size = size
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
@@ -425,7 +426,8 @@
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
         buffer = self._get_buffer()
-        return visitor.visit_vrawbuffer(self.size,
+        return visitor.visit_vrawbuffer(self.func,
+                                        self.size,
                                         buffer.offsets[:],
                                         buffer.descrs[:])
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1770,7 +1770,7 @@
     def test_virtual_raw_malloc_basic(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)   # 12345 = malloc func
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1787,7 +1787,7 @@
         ops = """
         [i1]
         i5 = int_mul(10, 1)
-        i2 = call_i('malloc', i5, descr=raw_malloc_descr)
+        i2 = call_i(12345, i5, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1803,7 +1803,7 @@
     def test_virtual_raw_malloc_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1817,7 +1817,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr_char)
         raw_store(i2, 1, 123, descr=rawarraydescr_char)
@@ -1832,7 +1832,7 @@
     def test_virtual_raw_malloc_invalid_write_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1843,7 +1843,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1855,7 +1855,7 @@
     def test_virtual_raw_malloc_invalid_read_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1866,7 +1866,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr_char)
@@ -1878,7 +1878,7 @@
     def test_virtual_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1898,7 +1898,7 @@
     def test_virtual_raw_slice_of_a_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         i3 = int_add(i2, 1) # get a slice of the original buffer
         i4 = int_add(i3, 1) # get a slice of a slice
@@ -1916,7 +1916,7 @@
     def test_virtual_raw_slice_force(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1929,7 +1929,7 @@
         [i0, i1]
         label(i0, i1)
         # these ops are generated by VirtualRawBufferValue._really_force
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, 42, descr=rawarraydescr_char)
         raw_store(i2, 5, 4242, descr=rawarraydescr_char)
@@ -1946,7 +1946,7 @@
         i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr)
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i3, 0, i2, descr=rawarraydescr)
         label(i2)
@@ -1958,7 +1958,7 @@
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
         label(i2)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i3)
         raw_store(i3, 0, i2, descr=rawarraydescr)
         jump(i3)
@@ -1968,7 +1968,7 @@
     def test_virtual_raw_store_raw_load(self):
         ops = """
         [i1]
-        i0 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i0 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 0, i1, descr=rawarraydescr)
         i2 = raw_load_i(i0, 0, descr=rawarraydescr)
@@ -1986,7 +1986,7 @@
     def test_virtual_raw_store_getarrayitem_raw(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 8, f1, descr=rawarraydescr_float)
         f2 = getarrayitem_raw_f(i0, 1, descr=rawarraydescr_float)
@@ -2004,7 +2004,7 @@
     def test_virtual_setarrayitem_raw_raw_load(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i0, 1, f1, descr=rawarraydescr_float)
         f2 = raw_load_f(i0, 8, descr=rawarraydescr_float)
@@ -2022,7 +2022,7 @@
     def test_virtual_raw_buffer_forced_but_slice_not_forced(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         i1 = int_add(i0, 8)
         escape_n(i0)
@@ -2031,7 +2031,7 @@
         """
         expected = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         check_memory_error(i0)
         escape_n(i0)
         i1 = int_add(i0, 8)
@@ -8886,7 +8886,7 @@
     def test_resume_forced_raw_ptr(self):
         ops = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         is = int_add(i, 8)
         escape_n(i)
@@ -8898,7 +8898,7 @@
         """
         expected = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i)
         escape_n(i)
         i1 = int_add(i0, 1)
@@ -8966,7 +8966,7 @@
     def test_pending_setfield_delayed_malloc(self):
         ops = """
         [i0, p0]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 13, descr=rawarraydescr)
         setfield_gc(p0, i2, descr=valuedescr)
@@ -8988,14 +8988,14 @@
     def test_raw_buffer_ptr_info_intbounds_bug(self):
         ops = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         guard_value(i2, 12345) []
         jump()
         """
         expected = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         guard_value(i2, 12345) []
         jump()
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -45,7 +45,8 @@
         return opinfo
 
     def make_virtual_raw_memory(self, size, source_op):
-        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, size)
+        func = source_op.getarg(0).getint()
+        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, func, size)
         newop = self.replace_op_with(source_op, source_op.getopnum(),
                                      args=[source_op.getarg(0), ConstInt(size)])
         newop.set_forwarded(opinfo)
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -364,8 +364,8 @@
     def visit_varraystruct(self, arraydescr, size, fielddescrs):
         return VArrayStructInfo(arraydescr, size, fielddescrs)
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
-        return VRawBufferInfo(size, offsets, descrs)
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
+        return VRawBufferInfo(func, size, offsets, descrs)
 
     def visit_vrawslice(self, offset):
         return VRawSliceInfo(offset)
@@ -703,7 +703,8 @@
 
 class VRawBufferInfo(VAbstractRawInfo):
 
-    def __init__(self, size, offsets, descrs):
+    def __init__(self, func, size, offsets, descrs):
+        self.func = func
         self.size = size
         self.offsets = offsets
         self.descrs = descrs
@@ -711,7 +712,7 @@
     @specialize.argtype(1)
     def allocate_int(self, decoder, index):
         length = len(self.fieldnums)
-        buffer = decoder.allocate_raw_buffer(self.size)
+        buffer = decoder.allocate_raw_buffer(self.func, self.size)
         decoder.virtuals_cache.set_int(index, buffer)
         for i in range(len(self.offsets)):
             offset = self.offsets[i]
@@ -1130,9 +1131,13 @@
                                                            lengthbox)
         return self.metainterp.execute_new_array(arraydescr, lengthbox)
 
-    def allocate_raw_buffer(self, size):
+    def allocate_raw_buffer(self, func, size):
         cic = self.metainterp.staticdata.callinfocollection
-        calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        # Can't use 'func' from callinfo_for_oopspec(), because we have
+        # several variants (zero/non-zero, memory-pressure or not, etc.)
+        # and we have to pick the correct one here; that's why we save
+        # it in the VRawBufferInfo.
         return self.metainterp.execute_and_record_varargs(
             rop.CALL_I, [ConstInt(func), ConstInt(size)], calldescr)
 
@@ -1461,10 +1466,11 @@
     def allocate_string(self, length):
         return self.cpu.bh_newstr(length)
 
-    def allocate_raw_buffer(self, size):
-        buffer = self.cpu.bh_new_raw_buffer(size)
-        adr = llmemory.cast_ptr_to_adr(buffer)
-        return llmemory.cast_adr_to_int(adr, "symbolic")
+    def allocate_raw_buffer(self, func, size):
+        from rpython.jit.codewriter import heaptracker
+        cic = self.callinfocollection
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        return self.cpu.bh_call_i(func, [size], None, None, calldescr)
 
     def string_setitem(self, str, index, charnum):
         char = self.decode_int(charnum)
diff --git a/rpython/jit/metainterp/walkvirtual.py b/rpython/jit/metainterp/walkvirtual.py
--- a/rpython/jit/metainterp/walkvirtual.py
+++ b/rpython/jit/metainterp/walkvirtual.py
@@ -17,7 +17,7 @@
     def visit_varraystruct(self, arraydescr, fielddescrs):
         raise NotImplementedError("abstract base class")
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
         raise NotImplementedError("abstract base class")
 
     def visit_vrawslice(self, offset):
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -452,7 +452,8 @@
                               merge_if_blocks=True,
                               constfold=True,
                               remove_asserts=True,
-                              really_remove_asserts=True)
+                              really_remove_asserts=True,
+                              replace_we_are_jitted=False)
 
     def prejit_optimizations_minimal_inline(self, policy, graphs):
         from rpython.translator.backendopt.inline import auto_inline_graphs
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -799,8 +799,8 @@
                            rffi.cast(size_t, map_size),
                            rffi.cast(rffi.INT, use_flag))
     else:
-        def madvice_free(addr, map_size):
-            "No madvice() on this platform"
+        def madvise_free(addr, map_size):
+            "No madvise() on this platform"
 
 elif _MS_WINDOWS:
     def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -250,6 +250,7 @@
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
     HAVE_UTIMES = rffi_platform.Has('utimes')
+    HAVE_D_TYPE = rffi_platform.Has('DT_UNKNOWN')
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
@@ -603,23 +604,34 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))])
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
+        if HAVE_D_TYPE:
+            DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
+            DT_REG     = rffi_platform.ConstantInteger('DT_REG')
+            DT_DIR     = rffi_platform.ConstantInteger('DT_DIR')
+            DT_LNK     = rffi_platform.ConstantInteger('DT_LNK')
 
     DIRP = rffi.COpaquePtr('DIR')
-    config = rffi_platform.configure(CConfig)
-    DIRENT = config['DIRENT']
+    dirent_config = rffi_platform.configure(CConfig)
+    DIRENT = dirent_config['DIRENT']
     DIRENTP = lltype.Ptr(DIRENT)
     c_opendir = external('opendir',
         [rffi.CCHARP], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
     c_fdopendir = external('fdopendir',
         [rffi.INT], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
+    c_rewinddir = external('rewinddir',
+        [DIRP], lltype.Void, releasegil=False)
     # XXX macro=True is hack to make sure we get the correct kind of
     # dirent struct (which depends on defines)
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
-    c_closedir = external('closedir', [DIRP], rffi.INT)
+    c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
+    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+else:
+    dirent_config = {}
 
-def _listdir(dirp):
+def _listdir(dirp, rewind=False):
     result = []
     while True:
         direntp = c_readdir(dirp)
@@ -630,6 +642,8 @@
         name = rffi.charp2str(namep)
         if name != '.' and name != '..':
             result.append(name)
+    if rewind:
+        c_rewinddir(dirp)
     c_closedir(dirp)
     if error:
         raise OSError(error, "readdir failed")
@@ -640,12 +654,16 @@
     Like listdir(), except that the directory is specified as an open
     file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.
+    Note: fdlistdir() closes the file descriptor.  To emulate the
+    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+    file descriptor.
     """
     dirp = c_fdopendir(dirfd)
     if not dirp:
-        raise OSError(get_saved_errno(), "opendir failed")
-    return _listdir(dirp)
+        error = get_saved_errno()
+        c_close(dirfd)
+        raise OSError(error, "opendir failed")
+    return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)
@@ -1782,12 +1800,7 @@
 # Support for f... and ...at families of POSIX functions
 
 class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes=['sys/stat.h',
-                  'sys/time.h',
-                  'unistd.h',
-                  'fcntl.h'],
-    )
+    _compilation_info_ = eci
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
             fdopendir fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rposix_scandir.py
@@ -0,0 +1,52 @@
+from rpython.rlib import rposix
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+ at specialize.argtype(0)
+def opendir(path):
+    path = rposix._as_bytes0(path)
+    return opendir_bytes(path)
+
+def opendir_bytes(path):
+    dirp = rposix.c_opendir(path)
+    if not dirp:
+        raise OSError(rposix.get_saved_errno(), "opendir failed")
+    return dirp
+
+def closedir(dirp):
+    rposix.c_closedir(dirp)
+
+NULL_DIRP = lltype.nullptr(rposix.DIRP.TO)
+
+def nextentry(dirp):
+    """Read the next entry and returns an opaque object.
+    Use the methods has_xxx() and get_xxx() to read from that
+    opaque object.  The opaque object is valid until the next
+    time nextentry() or closedir() is called.  This may raise
+    OSError, or return a NULL pointer when exhausted.  Note
+    that this doesn't filter out the "." and ".." entries.
+    """
+    direntp = rposix.c_readdir(dirp)
+    if direntp:
+        error = rposix.get_saved_errno()
+        if error:
+            raise OSError(error, "readdir failed")
+    return direntp
+
+def has_name_bytes(direntp):
+    return True
+
+def get_name_bytes(direntp):
+    namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
+    return rffi.charp2str(namep)
+
+DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0)
+DT_REG = rposix.dirent_config.get('DT_REG', 255)
+DT_DIR = rposix.dirent_config.get('DT_DIR', 255)
+DT_LNK = rposix.dirent_config.get('DT_LNK', 255)
+
+def get_known_type(direntp):
+    if rposix.HAVE_D_TYPE:
+        return rffi.getintfield(direntp, 'c_d_type')
+    return DT_UNKNOWN
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -90,6 +90,9 @@
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
+        attrs = CodeClass.__dict__.get('_attrs_', None)
+        if attrs is not None:
+            CodeClass._attrs_ = list(attrs) + ['_vmprof_unique_id']
         self._code_classes.add(CodeClass)
         #
         class WeakCodeObjectList(RWeakListMixin):
@@ -189,7 +192,7 @@
 
         def decorated_function(*args):
             unique_id = get_code_fn(*args)._vmprof_unique_id
-            unique_id = rffi.cast(lltype.Signed, unique_id) 
+            unique_id = rffi.cast(lltype.Signed, unique_id)
             # ^^^ removes the "known non-negative" hint for annotation
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -552,6 +552,14 @@
     # Note: fdlistdir() always closes dirfd
     assert result == ['file']
 
+ at rposix_requires('fdlistdir')
+def test_fdlistdir_rewinddir(tmpdir):
+    tmpdir.join('file').write('text')
+    dirfd = os.open(str(tmpdir), os.O_RDONLY)
+    result1 = rposix.fdlistdir(os.dup(dirfd))
+    result2 = rposix.fdlistdir(dirfd)
+    assert result1 == result2 == ['file']
+
 @rposix_requires('symlinkat')
 def test_symlinkat(tmpdir):
     tmpdir.join('file').write('text')
diff --git a/rpython/rlib/test/test_rposix_scandir.py b/rpython/rlib/test/test_rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rposix_scandir.py
@@ -0,0 +1,21 @@
+import sys, os
+import py
+from rpython.rlib import rposix_scandir
+
+
+class TestScanDir(object):
+
+    @py.test.mark.skipif("sys.platform == 'win32'")   # XXX
+    def test_name_bytes(self):
+        scan = rposix_scandir.opendir('/')
+        found = []
+        while True:
+            p = rposix_scandir.nextentry(scan)
+            if not p:
+                break
+            assert rposix_scandir.has_name_bytes(p)
+            found.append(rposix_scandir.get_name_bytes(p))
+        rposix_scandir.closedir(scan)
+        found.remove('.')
+        found.remove('..')
+        assert sorted(found) == sorted(os.listdir('/'))
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -648,6 +648,7 @@
 
     @staticmethod
     @jit.elidable
+    @signature(types.any(), types.any(), types.int(), types.int(), returns=types.int())
     def ll_rfind_char(s, ch, start, end):
         if end > len(s.chars):
             end = len(s.chars)
diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py
--- a/rpython/translator/backendopt/all.py
+++ b/rpython/translator/backendopt/all.py
@@ -2,6 +2,7 @@
 from rpython.translator.backendopt import inline
 from rpython.translator.backendopt.malloc import remove_mallocs
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.translator.backendopt.stat import print_statistics
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator import simplify
@@ -36,6 +37,7 @@
     # inline_threshold, mallocs
     # merge_if_blocks, constfold, heap2stack
     # clever_malloc_removal, remove_asserts
+    # replace_we_are_jitted
 
     config = translator.config.translation.backendopt.copy(as_default=True)
     config.set(**kwds)
@@ -49,6 +51,10 @@
         print "before optimizations:"
         print_statistics(translator.graphs[0], translator, "per-graph.txt")
 
+    if config.replace_we_are_jitted:
+        for graph in graphs:
+            replace_we_are_jitted(graph)
+
     if config.remove_asserts:
         constfold(config, graphs)
         remove_asserts(translator, graphs)
diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py
--- a/rpython/translator/backendopt/constfold.py
+++ b/rpython/translator/backendopt/constfold.py
@@ -276,3 +276,25 @@
             rewire_links(splitblocks, graph)
         if not diffused and not splitblocks:
             break # finished
+
+def replace_symbolic(graph, symbolic, value):
+    result = False
+    for block in graph.iterblocks():
+        for op in block.operations:
+            for i, arg in enumerate(op.args):
+                if isinstance(arg, Constant) and arg.value is symbolic:
+                    op.args[i] = value
+                    result = True
+        if block.exitswitch is symbolic:
+            block.exitswitch = value
+            result = True
+    return result
+
+def replace_we_are_jitted(graph):
+    from rpython.rlib import jit
+    replacement = Constant(0)
+    replacement.concretetype = lltype.Signed
+    did_replacement = replace_symbolic(graph, jit._we_are_jitted, replacement)
+    if did_replacement:
+        constant_fold_graph(graph)
+    return did_replacement
diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py
--- a/rpython/translator/backendopt/test/test_all.py
+++ b/rpython/translator/backendopt/test/test_all.py
@@ -289,3 +289,19 @@
         llinterp = LLInterpreter(t.rtyper)
         res = llinterp.eval_graph(later_graph, [10])
         assert res == 1
+
+    def test_replace_we_are_jitted(self):
+        from rpython.rlib import jit
+        def f():
+            if jit.we_are_jitted():
+                return 1
+            return 2 + jit.we_are_jitted()
+
+        t = self.translateopt(f, [])
+        graph = graphof(t, f)
+        # by default, replace_we_are_jitted is off
+        assert graph.startblock.operations[0].args[0].value is jit._we_are_jitted
+
+        t = self.translateopt(f, [], replace_we_are_jitted=True)
+        graph = graphof(t, f)
+        assert graph.startblock.exits[0].args[0].value == 2
diff --git a/rpython/translator/backendopt/test/test_constfold.py b/rpython/translator/backendopt/test/test_constfold.py
--- a/rpython/translator/backendopt/test/test_constfold.py
+++ b/rpython/translator/backendopt/test/test_constfold.py
@@ -7,6 +7,7 @@
 from rpython.rtyper import rclass
 from rpython.rlib import objectmodel
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.conftest import option
 
 def get_graph(fn, signature):
@@ -343,3 +344,18 @@
     merge_if_blocks.merge_if_blocks_once(graph)
     constant_fold_graph(graph)
     check_graph(graph, [], 66, t)
+
+def test_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def fn():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+    graph, t = get_graph(fn, [])
+    result = replace_we_are_jitted(graph)
+    assert result
+    checkgraph(graph)
+    # check shape of graph
+    assert len(graph.startblock.operations) == 0
+    assert graph.startblock.exitswitch is None
+    assert graph.startblock.exits[0].target.exits[0].args[0].value == 2
diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -381,7 +381,7 @@
         """ Run all backend optimizations - lltype version
         """
         from rpython.translator.backendopt.all import backend_optimizations
-        backend_optimizations(self.translator)
+        backend_optimizations(self.translator, replace_we_are_jitted=True)
 
 
     STACKCHECKINSERTION = 'stackcheckinsertion_lltype'
diff --git a/rpython/translator/test/test_interactive.py b/rpython/translator/test/test_interactive.py
--- a/rpython/translator/test/test_interactive.py
+++ b/rpython/translator/test/test_interactive.py
@@ -78,3 +78,15 @@
     dll = ctypes.CDLL(str(t.driver.c_entryp))
     f = dll.pypy_g_f
     assert f(2, 3) == 5
+
+def test_check_that_driver_uses_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def f():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+
+    t = Translation(f, [])
+    t.backendopt()
+    graph = t.driver.translator.graphs[0]
+    assert graph.startblock.exits[0].args[0].value == 2

From pypy.commits at gmail.com  Tue Aug 23 11:27:55 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 23 Aug 2016 08:27:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Make sure that hash(x) == x.__hash__()
 for int, long
Message-ID: <57bc6b7b.041f1c0a.6f5fb.8ba1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86451:eeb7fd8a6676
Date: 2016-08-23 16:27 +0100
http://bitbucket.org/pypy/pypy/changeset/eeb7fd8a6676/

Log:	Make sure that hash(x) == x.__hash__() for int, long

diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -362,11 +362,10 @@
         return _new_int(space, w_inttype, w_x, w_base)
 
     def descr_hash(self, space):
-        # unlike CPython, we don't special-case the value -1 in most of
-        # our hash functions, so there is not much sense special-casing
-        # it here either.  Make sure this is consistent with the hash of
-        # floats and longs.
-        return self.int(space)
+        # For compatibility with CPython, we special-case -1
+        h = self.intval
+        h -= (h == -1)  # No explicit condition, to avoid JIT bridges
+        return wrapint(space, h)
 
     def _int(self, space):
         return self.int(space)
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -130,7 +130,12 @@
 
     descr_repr = _make_descr_unaryop('repr')
     descr_str = _make_descr_unaryop('str')
-    descr_hash = _make_descr_unaryop('hash')
+
+    def descr_hash(self, space):
+        h = self.asbigint().hash()
+        h -= (h == -1)
+        return space.newint(h)
+
     descr_oct = _make_descr_unaryop('oct')
     descr_hex = _make_descr_unaryop('hex')
 
@@ -387,12 +392,12 @@
     def _make_generic_descr_binop(opname):
         if opname not in COMMUTATIVE_OPS:
             raise Exception("Not supported")
-            
+
         methname = opname + '_' if opname in ('and', 'or') else opname
         descr_rname = 'descr_r' + opname
         op = getattr(rbigint, methname)
         intop = getattr(rbigint, "int_" + methname)
-        
+
         @func_renamer('descr_' + opname)
         def descr_binop(self, space, w_other):
             if isinstance(w_other, W_AbstractIntObject):
@@ -412,7 +417,7 @@
             return W_LongObject(op(w_other.asbigint(), self.num))
 
         return descr_binop, descr_rbinop
-        
+
     descr_add, descr_radd = _make_generic_descr_binop('add')
     descr_sub, descr_rsub = _make_generic_descr_binop_noncommutative('sub')
     descr_mul, descr_rmul = _make_generic_descr_binop('mul')
@@ -454,12 +459,12 @@
         except OverflowError:   # b too big
             raise oefmt(space.w_OverflowError, "shift count too large")
         return W_LongObject(self.num.lshift(shift))
-        
+
     def _int_lshift(self, space, w_other):
         if w_other < 0:
             raise oefmt(space.w_ValueError, "negative shift count")
         return W_LongObject(self.num.lshift(w_other))
-        
+
     descr_lshift, descr_rlshift = _make_descr_binop(_lshift, _int_lshift)
 
     def _rshift(self, space, w_other):
@@ -470,7 +475,7 @@
         except OverflowError:   # b too big # XXX maybe just return 0L instead?
             raise oefmt(space.w_OverflowError, "shift count too large")
         return newlong(space, self.num.rshift(shift))
-        
+
     def _int_rshift(self, space, w_other):
         if w_other < 0:
             raise oefmt(space.w_ValueError, "negative shift count")
@@ -485,7 +490,7 @@
             raise oefmt(space.w_ZeroDivisionError,
                         "long division or modulo by zero")
         return newlong(space, z)
-        
+
     def _floordiv(self, space, w_other):
         try:
             z = self.num.floordiv(w_other.asbigint())
@@ -505,7 +510,7 @@
             raise oefmt(space.w_ZeroDivisionError,
                         "long division or modulo by zero")
         return newlong(space, z)
-        
+
     def _int_mod(self, space, w_other):
         try:
             z = self.num.int_mod(w_other)
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -295,7 +295,11 @@
         assert self.space.unwrap(result) == hex(x)
 
 
-class AppTestInt:
+class AppTestInt(object):
+    def test_hash(self):
+        assert hash(-1) == (-1).__hash__() == -2
+        assert hash(-2) == (-2).__hash__() == -2
+
     def test_conjugate(self):
         assert (1).conjugate() == 1
         assert (-1).conjugate() == -1
@@ -454,11 +458,11 @@
                 return None
         inst = a()
         raises(TypeError, int, inst)
-        assert inst.ar == True 
+        assert inst.ar == True
 
         class b(object):
-            pass 
-        raises((AttributeError,TypeError), int, b()) 
+            pass
+        raises((AttributeError,TypeError), int, b())
 
     def test_special_long(self):
         class a(object):
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -228,7 +228,7 @@
     def test_hash(self):
         # ints have the same hash as equal longs
         for i in range(-4, 14):
-            assert hash(i) == hash(long(i))
+            assert hash(i) == hash(long(i)) == long(i).__hash__()
         # might check too much -- it's ok to change the hashing algorithm
         assert hash(123456789L) == 123456789
         assert hash(1234567890123456789L) in (

From pypy.commits at gmail.com  Tue Aug 23 11:43:02 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 23 Aug 2016 08:43:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Describe key-value reverse order bug for
 dicts
Message-ID: <57bc6f06.c62f1c0a.9f43d.8569@mx.google.com>

Author: Raffael Tfirst <raffael.tfirst at gmail.com>
Branch: py3.5
Changeset: r86452:36436e3e01af
Date: 2016-08-23 17:42 +0200
http://bitbucket.org/pypy/pypy/changeset/36436e3e01af/

Log:	Describe key-value reverse order bug for dicts

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1246,8 +1246,14 @@
                     d.values[i].walkabout(self)
                     containers += 1
                 else:
+                    # TODO: key.walkabout has to be called before d.values.walkabout
+                    # that would fix the error "keywords must be strings"
+                    # for some reason the keys and values seem to be in reverse order
+                    # in some cases, so another error has to be fixed in order for
+                    # this to work, otherwise it breaks everything
+                    # after fix: remove dirty fixes in pyopcode
                     d.values[i].walkabout(self)
-                    d.keys[i].walkabout(self)
+                    key.walkabout(self)
                     elements += 1
         if elements or containers == 0:
             self.emit_op_arg(ops.BUILD_MAP, elements)

From pypy.commits at gmail.com  Tue Aug 23 11:48:41 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 23 Aug 2016 08:48:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57bc7059.4152c20a.199e7.58d4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86453:36d4233f4e5c
Date: 2016-08-23 16:46 +0100
http://bitbucket.org/pypy/pypy/changeset/36d4233f4e5c/

Log:	hg merge default

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -423,7 +423,7 @@
         guard_false(i114, descr=...)
         --TICK--
         i123 = arraylen_gc(p67, descr=<ArrayP .>)
-        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=5 OS=110>)
+        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=<Calli . i EF=5 OS=110>)
         check_memory_error(i119)
         raw_store(i119, 0, i160, descr=<ArrayS 2>)
         raw_store(i119, 2, i160, descr=<ArrayS 2>)
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1029,4 +1029,5 @@
     if x >= HASH_MODULUS:
         x -= HASH_MODULUS
 
-    return intmask(intmask(x) * sign)
+    h = intmask(intmask(x) * sign)
+    return h - (h == -1)
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -254,12 +254,12 @@
     def _make_generic_descr_binop(opname):
         if opname not in COMMUTATIVE_OPS:
             raise Exception("Not supported")
-            
+
         methname = opname + '_' if opname in ('and', 'or') else opname
         descr_rname = 'descr_r' + opname
         op = getattr(rbigint, methname)
         intop = getattr(rbigint, "int_" + methname)
-        
+
         @func_renamer('descr_' + opname)
         def descr_binop(self, space, w_other):
             if isinstance(w_other, W_IntObject):
@@ -279,7 +279,7 @@
             return W_LongObject(op(w_other.asbigint(), self.num))
 
         return descr_binop, descr_rbinop
-        
+
     descr_add, descr_radd = _make_generic_descr_binop('add')
     descr_sub, descr_rsub = _make_generic_descr_binop_noncommutative('sub')
     descr_mul, descr_rmul = _make_generic_descr_binop('mul')
@@ -321,12 +321,12 @@
         except OverflowError:   # b too big
             raise oefmt(space.w_OverflowError, "shift count too large")
         return W_LongObject(self.num.lshift(shift))
-        
+
     def _int_lshift(self, space, w_other):
         if w_other < 0:
             raise oefmt(space.w_ValueError, "negative shift count")
         return W_LongObject(self.num.lshift(w_other))
-        
+
     descr_lshift, descr_rlshift = _make_descr_binop(_lshift, _int_lshift)
 
     def _rshift(self, space, w_other):
@@ -337,7 +337,7 @@
         except OverflowError:   # b too big # XXX maybe just return 0L instead?
             raise oefmt(space.w_OverflowError, "shift count too large")
         return newlong(space, self.num.rshift(shift))
-        
+
     def _int_rshift(self, space, w_other):
         if w_other < 0:
             raise oefmt(space.w_ValueError, "negative shift count")
@@ -352,7 +352,7 @@
             raise oefmt(space.w_ZeroDivisionError,
                         "long division or modulo by zero")
         return newlong(space, z)
-        
+
     def _floordiv(self, space, w_other):
         try:
             z = self.num.floordiv(w_other.asbigint())
@@ -369,7 +369,7 @@
             raise oefmt(space.w_ZeroDivisionError,
                         "integer division or modulo by zero")
         return newlong(space, z)
-        
+
     def _int_mod(self, space, w_other):
         try:
             z = self.num.int_mod(w_other)
@@ -404,7 +404,8 @@
         while x >= HASH_MODULUS:
             x -= HASH_MODULUS
         i -= 1
-    return intmask(intmask(x) * v.sign)
+    h = intmask(intmask(x) * v.sign)
+    return h - (h == -1)
 
 
 def newlong(space, bigint):
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -265,7 +265,11 @@
         result = f1.int(self.space)
         assert result == f1
 
-class AppTestInt:
+class AppTestInt(object):
+    def test_hash(self):
+        assert hash(-1) == (-1).__hash__() == -2
+        assert hash(-2) == (-2).__hash__() == -2
+
     def test_conjugate(self):
         assert (1).conjugate() == 1
         assert (-1).conjugate() == -1

From pypy.commits at gmail.com  Tue Aug 23 12:03:33 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 23 Aug 2016 09:03:33 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Detect overflow errors in
 malloc_varsize_clear
Message-ID: <57bc73d5.4219c20a.987dd.57e4@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86454:92eeddd8d966
Date: 2016-08-23 18:02 +0200
http://bitbucket.org/pypy/pypy/changeset/92eeddd8d966/

Log:	Detect overflow errors in malloc_varsize_clear

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype, llgroup, llmemory, llarena
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert
+from rpython.rlib.rarithmetic import ovfcheck
 
 class QCGC(GCBase):
     _alloc_flavor_ = "raw"
@@ -43,8 +44,15 @@
 
     def malloc_varsize_clear(self, typeid, length, size, itemsize,
                              offset_to_length):
-        totalsize = size + itemsize * length
-        #totalsize = llarena.round_up_for_allocation(totalsize)
+        if length < 0:
+            raise MemoryError
+        #
+        try:
+            varsize = ovfcheck(itemsize * length)
+            totalsize = ovfcheck(size + varsize)
+        except OverflowError:
+            raise MemoryError
+        #
         obj = llop.qcgc_allocate(llmemory.Address, totalsize)
         self.init_gc_object(obj, typeid)
         (obj + offset_to_length).signed[0] = length

From pypy.commits at gmail.com  Wed Aug 24 03:14:18 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 00:14:18 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: merge default
Message-ID: <57bd494a.898b1c0a.1c93f.8cd7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86455:4ec423c97a35
Date: 2016-08-24 09:10 +0200
http://bitbucket.org/pypy/pypy/changeset/4ec423c97a35/

Log:	merge default

diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -144,3 +144,9 @@
 ``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
 Previously it returned what looked like a regular dict object (but it
 was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -590,6 +590,8 @@
                                                     "decode", w_fs_encoding)
                 except OperationError as e:
                     # fall back to the original byte string
+                    if e.async(space):
+                        raise
                     result_w[i] = w_bytes
             return space.newlist(result_w)
         else:
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -423,7 +423,7 @@
         guard_false(i114, descr=...)
         --TICK--
         i123 = arraylen_gc(p67, descr=<ArrayP .>)
-        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=5 OS=110>)
+        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=<Calli . i EF=5 OS=110>)
         check_memory_error(i119)
         raw_store(i119, 0, i160, descr=<ArrayS 2>)
         raw_store(i119, 2, i160, descr=<ArrayS 2>)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1963,3 +1963,21 @@
                        ffi, "test_function_returns_opaque", "?")
     assert str(e.value) == ("function foo: 'struct a' is used as result type,"
                             " but is opaque")
+
+def test_function_returns_union():
+    ffi = FFI()
+    ffi.cdef("union u1 { int a, b; }; union u1 f1(int);")
+    lib = verify(ffi, "test_function_returns_union", """
+        union u1 { int a, b; };
+        static union u1 f1(int x) { union u1 u; u.b = x; return u; }
+    """)
+    assert lib.f1(51).a == 51
+
+def test_function_returns_partial_struct():
+    ffi = FFI()
+    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    lib = verify(ffi, "test_function_returns_partial_struct", """
+        struct a { int b, a, c; };
+        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+    """)
+    assert lib.f1(52).a == 52
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -466,8 +466,6 @@
     def descr_getbuffer(self, space, w_flags):
         #from pypy.objspace.std.bufferobject import W_Buffer
         #return W_Buffer(StringBuffer(self._value))
-        # XXX handle flags, figure out why returning a W_Buffer
-        #     makes upstream ctypes tests fail
         return self
 
     charbuf_w = str_w
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -362,11 +362,10 @@
         return _new_int(space, w_inttype, w_x, w_base)
 
     def descr_hash(self, space):
-        # unlike CPython, we don't special-case the value -1 in most of
-        # our hash functions, so there is not much sense special-casing
-        # it here either.  Make sure this is consistent with the hash of
-        # floats and longs.
-        return self.int(space)
+        # For compatibility with CPython, we special-case -1
+        h = self.intval
+        h -= (h == -1)  # No explicit condition, to avoid JIT bridges
+        return wrapint(space, h)
 
     def _int(self, space):
         return self.int(space)
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -130,7 +130,12 @@
 
     descr_repr = _make_descr_unaryop('repr')
     descr_str = _make_descr_unaryop('str')
-    descr_hash = _make_descr_unaryop('hash')
+
+    def descr_hash(self, space):
+        h = self.asbigint().hash()
+        h -= (h == -1)
+        return space.newint(h)
+
     descr_oct = _make_descr_unaryop('oct')
     descr_hex = _make_descr_unaryop('hex')
 
@@ -387,12 +392,12 @@
     def _make_generic_descr_binop(opname):
         if opname not in COMMUTATIVE_OPS:
             raise Exception("Not supported")
-            
+
         methname = opname + '_' if opname in ('and', 'or') else opname
         descr_rname = 'descr_r' + opname
         op = getattr(rbigint, methname)
         intop = getattr(rbigint, "int_" + methname)
-        
+
         @func_renamer('descr_' + opname)
         def descr_binop(self, space, w_other):
             if isinstance(w_other, W_AbstractIntObject):
@@ -412,7 +417,7 @@
             return W_LongObject(op(w_other.asbigint(), self.num))
 
         return descr_binop, descr_rbinop
-        
+
     descr_add, descr_radd = _make_generic_descr_binop('add')
     descr_sub, descr_rsub = _make_generic_descr_binop_noncommutative('sub')
     descr_mul, descr_rmul = _make_generic_descr_binop('mul')
@@ -454,12 +459,12 @@
         except OverflowError:   # b too big
             raise oefmt(space.w_OverflowError, "shift count too large")
         return W_LongObject(self.num.lshift(shift))
-        
+
     def _int_lshift(self, space, w_other):
         if w_other < 0:
             raise oefmt(space.w_ValueError, "negative shift count")
         return W_LongObject(self.num.lshift(w_other))
-        
+
     descr_lshift, descr_rlshift = _make_descr_binop(_lshift, _int_lshift)
 
     def _rshift(self, space, w_other):
@@ -470,7 +475,7 @@
         except OverflowError:   # b too big # XXX maybe just return 0L instead?
             raise oefmt(space.w_OverflowError, "shift count too large")
         return newlong(space, self.num.rshift(shift))
-        
+
     def _int_rshift(self, space, w_other):
         if w_other < 0:
             raise oefmt(space.w_ValueError, "negative shift count")
@@ -485,7 +490,7 @@
             raise oefmt(space.w_ZeroDivisionError,
                         "long division or modulo by zero")
         return newlong(space, z)
-        
+
     def _floordiv(self, space, w_other):
         try:
             z = self.num.floordiv(w_other.asbigint())
@@ -505,7 +510,7 @@
             raise oefmt(space.w_ZeroDivisionError,
                         "long division or modulo by zero")
         return newlong(space, z)
-        
+
     def _int_mod(self, space, w_other):
         try:
             z = self.num.int_mod(w_other)
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -295,7 +295,11 @@
         assert self.space.unwrap(result) == hex(x)
 
 
-class AppTestInt:
+class AppTestInt(object):
+    def test_hash(self):
+        assert hash(-1) == (-1).__hash__() == -2
+        assert hash(-2) == (-2).__hash__() == -2
+
     def test_conjugate(self):
         assert (1).conjugate() == 1
         assert (-1).conjugate() == -1
@@ -454,11 +458,11 @@
                 return None
         inst = a()
         raises(TypeError, int, inst)
-        assert inst.ar == True 
+        assert inst.ar == True
 
         class b(object):
-            pass 
-        raises((AttributeError,TypeError), int, b()) 
+            pass
+        raises((AttributeError,TypeError), int, b())
 
     def test_special_long(self):
         class a(object):
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -228,7 +228,7 @@
     def test_hash(self):
         # ints have the same hash as equal longs
         for i in range(-4, 14):
-            assert hash(i) == hash(long(i))
+            assert hash(i) == hash(long(i)) == long(i).__hash__()
         # might check too much -- it's ok to change the hashing algorithm
         assert hash(123456789L) == 123456789
         assert hash(1234567890123456789L) in (
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -261,6 +261,9 @@
                    "stack based virtual machines (only for backends that support it)",
                    default=True),
         BoolOption("storesink", "Perform store sinking", default=True),
+        BoolOption("replace_we_are_jitted",
+                   "Replace we_are_jitted() calls by False",
+                   default=False, cmdline=None),
         BoolOption("none",
                    "Do not run any backend optimizations",
                    requires=[('translation.backendopt.inline', False),
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -833,9 +833,6 @@
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     # vector operations
     vector_arith_code = """
     def bh_vec_{0}_{1}(self, vx, vy, count):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -757,9 +757,6 @@
             self.write_int_at_mem(res, self.vtable_offset, WORD, sizedescr.get_vtable())
         return res
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -229,8 +229,6 @@
         raise NotImplementedError
     def bh_newunicode(self, length):
         raise NotImplementedError
-    def bh_new_raw_buffer(self, size):
-        raise NotImplementedError
 
     def bh_arraylen_gc(self, array, arraydescr):
         raise NotImplementedError
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -582,6 +582,14 @@
                 return lltype.malloc(ARRAY, n, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_1_raw_malloc_varsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_1_raw_malloc_varsize.func_name = name
             return _ll_1_raw_malloc_varsize
         return build_ll_1_raw_malloc_varsize
 
@@ -610,6 +618,14 @@
                 return lltype.malloc(STRUCT, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_0_raw_malloc_fixedsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_0_raw_malloc_fixedsize.func_name = name
             return _ll_0_raw_malloc_fixedsize
         return build_ll_0_raw_malloc_fixedsize
 
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -367,8 +367,9 @@
 
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
-    
-    def __init__(self, cpu, size=-1):
+
+    def __init__(self, cpu, func, size=-1):
+        self.func = func
         self.size = size
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
@@ -425,7 +426,8 @@
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
         buffer = self._get_buffer()
-        return visitor.visit_vrawbuffer(self.size,
+        return visitor.visit_vrawbuffer(self.func,
+                                        self.size,
                                         buffer.offsets[:],
                                         buffer.descrs[:])
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1770,7 +1770,7 @@
     def test_virtual_raw_malloc_basic(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)   # 12345 = malloc func
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1787,7 +1787,7 @@
         ops = """
         [i1]
         i5 = int_mul(10, 1)
-        i2 = call_i('malloc', i5, descr=raw_malloc_descr)
+        i2 = call_i(12345, i5, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1803,7 +1803,7 @@
     def test_virtual_raw_malloc_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1817,7 +1817,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr_char)
         raw_store(i2, 1, 123, descr=rawarraydescr_char)
@@ -1832,7 +1832,7 @@
     def test_virtual_raw_malloc_invalid_write_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1843,7 +1843,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1855,7 +1855,7 @@
     def test_virtual_raw_malloc_invalid_read_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1866,7 +1866,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr_char)
@@ -1878,7 +1878,7 @@
     def test_virtual_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1898,7 +1898,7 @@
     def test_virtual_raw_slice_of_a_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         i3 = int_add(i2, 1) # get a slice of the original buffer
         i4 = int_add(i3, 1) # get a slice of a slice
@@ -1916,7 +1916,7 @@
     def test_virtual_raw_slice_force(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1929,7 +1929,7 @@
         [i0, i1]
         label(i0, i1)
         # these ops are generated by VirtualRawBufferValue._really_force
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, 42, descr=rawarraydescr_char)
         raw_store(i2, 5, 4242, descr=rawarraydescr_char)
@@ -1946,7 +1946,7 @@
         i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr)
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i3, 0, i2, descr=rawarraydescr)
         label(i2)
@@ -1958,7 +1958,7 @@
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
         label(i2)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i3)
         raw_store(i3, 0, i2, descr=rawarraydescr)
         jump(i3)
@@ -1968,7 +1968,7 @@
     def test_virtual_raw_store_raw_load(self):
         ops = """
         [i1]
-        i0 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i0 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 0, i1, descr=rawarraydescr)
         i2 = raw_load_i(i0, 0, descr=rawarraydescr)
@@ -1986,7 +1986,7 @@
     def test_virtual_raw_store_getarrayitem_raw(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 8, f1, descr=rawarraydescr_float)
         f2 = getarrayitem_raw_f(i0, 1, descr=rawarraydescr_float)
@@ -2004,7 +2004,7 @@
     def test_virtual_setarrayitem_raw_raw_load(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i0, 1, f1, descr=rawarraydescr_float)
         f2 = raw_load_f(i0, 8, descr=rawarraydescr_float)
@@ -2022,7 +2022,7 @@
     def test_virtual_raw_buffer_forced_but_slice_not_forced(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         i1 = int_add(i0, 8)
         escape_n(i0)
@@ -2031,7 +2031,7 @@
         """
         expected = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         check_memory_error(i0)
         escape_n(i0)
         i1 = int_add(i0, 8)
@@ -8886,7 +8886,7 @@
     def test_resume_forced_raw_ptr(self):
         ops = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         is = int_add(i, 8)
         escape_n(i)
@@ -8898,7 +8898,7 @@
         """
         expected = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i)
         escape_n(i)
         i1 = int_add(i0, 1)
@@ -8966,7 +8966,7 @@
     def test_pending_setfield_delayed_malloc(self):
         ops = """
         [i0, p0]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 13, descr=rawarraydescr)
         setfield_gc(p0, i2, descr=valuedescr)
@@ -8988,14 +8988,14 @@
     def test_raw_buffer_ptr_info_intbounds_bug(self):
         ops = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         guard_value(i2, 12345) []
         jump()
         """
         expected = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         guard_value(i2, 12345) []
         jump()
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -45,7 +45,8 @@
         return opinfo
 
     def make_virtual_raw_memory(self, size, source_op):
-        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, size)
+        func = source_op.getarg(0).getint()
+        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, func, size)
         newop = self.replace_op_with(source_op, source_op.getopnum(),
                                      args=[source_op.getarg(0), ConstInt(size)])
         newop.set_forwarded(opinfo)
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -364,8 +364,8 @@
     def visit_varraystruct(self, arraydescr, size, fielddescrs):
         return VArrayStructInfo(arraydescr, size, fielddescrs)
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
-        return VRawBufferInfo(size, offsets, descrs)
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
+        return VRawBufferInfo(func, size, offsets, descrs)
 
     def visit_vrawslice(self, offset):
         return VRawSliceInfo(offset)
@@ -703,7 +703,8 @@
 
 class VRawBufferInfo(VAbstractRawInfo):
 
-    def __init__(self, size, offsets, descrs):
+    def __init__(self, func, size, offsets, descrs):
+        self.func = func
         self.size = size
         self.offsets = offsets
         self.descrs = descrs
@@ -711,7 +712,7 @@
     @specialize.argtype(1)
     def allocate_int(self, decoder, index):
         length = len(self.fieldnums)
-        buffer = decoder.allocate_raw_buffer(self.size)
+        buffer = decoder.allocate_raw_buffer(self.func, self.size)
         decoder.virtuals_cache.set_int(index, buffer)
         for i in range(len(self.offsets)):
             offset = self.offsets[i]
@@ -1130,9 +1131,13 @@
                                                            lengthbox)
         return self.metainterp.execute_new_array(arraydescr, lengthbox)
 
-    def allocate_raw_buffer(self, size):
+    def allocate_raw_buffer(self, func, size):
         cic = self.metainterp.staticdata.callinfocollection
-        calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        # Can't use 'func' from callinfo_for_oopspec(), because we have
+        # several variants (zero/non-zero, memory-pressure or not, etc.)
+        # and we have to pick the correct one here; that's why we save
+        # it in the VRawBufferInfo.
         return self.metainterp.execute_and_record_varargs(
             rop.CALL_I, [ConstInt(func), ConstInt(size)], calldescr)
 
@@ -1461,10 +1466,11 @@
     def allocate_string(self, length):
         return self.cpu.bh_newstr(length)
 
-    def allocate_raw_buffer(self, size):
-        buffer = self.cpu.bh_new_raw_buffer(size)
-        adr = llmemory.cast_ptr_to_adr(buffer)
-        return llmemory.cast_adr_to_int(adr, "symbolic")
+    def allocate_raw_buffer(self, func, size):
+        from rpython.jit.codewriter import heaptracker
+        cic = self.callinfocollection
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        return self.cpu.bh_call_i(func, [size], None, None, calldescr)
 
     def string_setitem(self, str, index, charnum):
         char = self.decode_int(charnum)
diff --git a/rpython/jit/metainterp/walkvirtual.py b/rpython/jit/metainterp/walkvirtual.py
--- a/rpython/jit/metainterp/walkvirtual.py
+++ b/rpython/jit/metainterp/walkvirtual.py
@@ -17,7 +17,7 @@
     def visit_varraystruct(self, arraydescr, fielddescrs):
         raise NotImplementedError("abstract base class")
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
         raise NotImplementedError("abstract base class")
 
     def visit_vrawslice(self, offset):
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -452,7 +452,8 @@
                               merge_if_blocks=True,
                               constfold=True,
                               remove_asserts=True,
-                              really_remove_asserts=True)
+                              really_remove_asserts=True,
+                              replace_we_are_jitted=False)
 
     def prejit_optimizations_minimal_inline(self, policy, graphs):
         from rpython.translator.backendopt.inline import auto_inline_graphs
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -799,8 +799,8 @@
                            rffi.cast(size_t, map_size),
                            rffi.cast(rffi.INT, use_flag))
     else:
-        def madvice_free(addr, map_size):
-            "No madvice() on this platform"
+        def madvise_free(addr, map_size):
+            "No madvise() on this platform"
 
 elif _MS_WINDOWS:
     def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -250,6 +250,7 @@
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
     HAVE_UTIMES = rffi_platform.Has('utimes')
+    HAVE_D_TYPE = rffi_platform.Has('DT_UNKNOWN')
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
@@ -603,23 +604,34 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))])
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
+        if HAVE_D_TYPE:
+            DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
+            DT_REG     = rffi_platform.ConstantInteger('DT_REG')
+            DT_DIR     = rffi_platform.ConstantInteger('DT_DIR')
+            DT_LNK     = rffi_platform.ConstantInteger('DT_LNK')
 
     DIRP = rffi.COpaquePtr('DIR')
-    config = rffi_platform.configure(CConfig)
-    DIRENT = config['DIRENT']
+    dirent_config = rffi_platform.configure(CConfig)
+    DIRENT = dirent_config['DIRENT']
     DIRENTP = lltype.Ptr(DIRENT)
     c_opendir = external('opendir',
         [rffi.CCHARP], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
     c_fdopendir = external('fdopendir',
         [rffi.INT], DIRP, save_err=rffi.RFFI_SAVE_ERRNO)
+    c_rewinddir = external('rewinddir',
+        [DIRP], lltype.Void, releasegil=False)
     # XXX macro=True is hack to make sure we get the correct kind of
     # dirent struct (which depends on defines)
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
-    c_closedir = external('closedir', [DIRP], rffi.INT)
+    c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
+    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+else:
+    dirent_config = {}
 
-def _listdir(dirp):
+def _listdir(dirp, rewind=False):
     result = []
     while True:
         direntp = c_readdir(dirp)
@@ -630,6 +642,8 @@
         name = rffi.charp2str(namep)
         if name != '.' and name != '..':
             result.append(name)
+    if rewind:
+        c_rewinddir(dirp)
     c_closedir(dirp)
     if error:
         raise OSError(error, "readdir failed")
@@ -640,12 +654,16 @@
     Like listdir(), except that the directory is specified as an open
     file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.
+    Note: fdlistdir() closes the file descriptor.  To emulate the
+    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+    file descriptor.
     """
     dirp = c_fdopendir(dirfd)
     if not dirp:
-        raise OSError(get_saved_errno(), "opendir failed")
-    return _listdir(dirp)
+        error = get_saved_errno()
+        c_close(dirfd)
+        raise OSError(error, "opendir failed")
+    return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)
@@ -1782,12 +1800,7 @@
 # Support for f... and ...at families of POSIX functions
 
 class CConfig:
-    _compilation_info_ = ExternalCompilationInfo(
-        includes=['sys/stat.h',
-                  'sys/time.h',
-                  'unistd.h',
-                  'fcntl.h'],
-    )
+    _compilation_info_ = eci
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
             fdopendir fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rposix_scandir.py
@@ -0,0 +1,52 @@
+from rpython.rlib import rposix
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+ at specialize.argtype(0)
+def opendir(path):
+    path = rposix._as_bytes0(path)
+    return opendir_bytes(path)
+
+def opendir_bytes(path):
+    dirp = rposix.c_opendir(path)
+    if not dirp:
+        raise OSError(rposix.get_saved_errno(), "opendir failed")
+    return dirp
+
+def closedir(dirp):
+    rposix.c_closedir(dirp)
+
+NULL_DIRP = lltype.nullptr(rposix.DIRP.TO)
+
+def nextentry(dirp):
+    """Read the next entry and returns an opaque object.
+    Use the methods has_xxx() and get_xxx() to read from that
+    opaque object.  The opaque object is valid until the next
+    time nextentry() or closedir() is called.  This may raise
+    OSError, or return a NULL pointer when exhausted.  Note
+    that this doesn't filter out the "." and ".." entries.
+    """
+    direntp = rposix.c_readdir(dirp)
+    if direntp:
+        error = rposix.get_saved_errno()
+        if error:
+            raise OSError(error, "readdir failed")
+    return direntp
+
+def has_name_bytes(direntp):
+    return True
+
+def get_name_bytes(direntp):
+    namep = rffi.cast(rffi.CCHARP, direntp.c_d_name)
+    return rffi.charp2str(namep)
+
+DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0)
+DT_REG = rposix.dirent_config.get('DT_REG', 255)
+DT_DIR = rposix.dirent_config.get('DT_DIR', 255)
+DT_LNK = rposix.dirent_config.get('DT_LNK', 255)
+
+def get_known_type(direntp):
+    if rposix.HAVE_D_TYPE:
+        return rffi.getintfield(direntp, 'c_d_type')
+    return DT_UNKNOWN
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -90,6 +90,9 @@
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
+        attrs = CodeClass.__dict__.get('_attrs_', None)
+        if attrs is not None:
+            CodeClass._attrs_ = list(attrs) + ['_vmprof_unique_id']
         self._code_classes.add(CodeClass)
         #
         class WeakCodeObjectList(RWeakListMixin):
@@ -189,7 +192,7 @@
 
         def decorated_function(*args):
             unique_id = get_code_fn(*args)._vmprof_unique_id
-            unique_id = rffi.cast(lltype.Signed, unique_id) 
+            unique_id = rffi.cast(lltype.Signed, unique_id)
             # ^^^ removes the "known non-negative" hint for annotation
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -552,6 +552,14 @@
     # Note: fdlistdir() always closes dirfd
     assert result == ['file']
 
+ at rposix_requires('fdlistdir')
+def test_fdlistdir_rewinddir(tmpdir):
+    tmpdir.join('file').write('text')
+    dirfd = os.open(str(tmpdir), os.O_RDONLY)
+    result1 = rposix.fdlistdir(os.dup(dirfd))
+    result2 = rposix.fdlistdir(dirfd)
+    assert result1 == result2 == ['file']
+
 @rposix_requires('symlinkat')
 def test_symlinkat(tmpdir):
     tmpdir.join('file').write('text')
diff --git a/rpython/rlib/test/test_rposix_scandir.py b/rpython/rlib/test/test_rposix_scandir.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rposix_scandir.py
@@ -0,0 +1,21 @@
+import sys, os
+import py
+from rpython.rlib import rposix_scandir
+
+
+class TestScanDir(object):
+
+    @py.test.mark.skipif("sys.platform == 'win32'")   # XXX
+    def test_name_bytes(self):
+        scan = rposix_scandir.opendir('/')
+        found = []
+        while True:
+            p = rposix_scandir.nextentry(scan)
+            if not p:
+                break
+            assert rposix_scandir.has_name_bytes(p)
+            found.append(rposix_scandir.get_name_bytes(p))
+        rposix_scandir.closedir(scan)
+        found.remove('.')
+        found.remove('..')
+        assert sorted(found) == sorted(os.listdir('/'))
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -648,6 +648,7 @@
 
     @staticmethod
     @jit.elidable
+    @signature(types.any(), types.any(), types.int(), types.int(), returns=types.int())
     def ll_rfind_char(s, ch, start, end):
         if end > len(s.chars):
             end = len(s.chars)
diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py
--- a/rpython/translator/backendopt/all.py
+++ b/rpython/translator/backendopt/all.py
@@ -2,6 +2,7 @@
 from rpython.translator.backendopt import inline
 from rpython.translator.backendopt.malloc import remove_mallocs
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.translator.backendopt.stat import print_statistics
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator import simplify
@@ -36,6 +37,7 @@
     # inline_threshold, mallocs
     # merge_if_blocks, constfold, heap2stack
     # clever_malloc_removal, remove_asserts
+    # replace_we_are_jitted
 
     config = translator.config.translation.backendopt.copy(as_default=True)
     config.set(**kwds)
@@ -49,6 +51,10 @@
         print "before optimizations:"
         print_statistics(translator.graphs[0], translator, "per-graph.txt")
 
+    if config.replace_we_are_jitted:
+        for graph in graphs:
+            replace_we_are_jitted(graph)
+
     if config.remove_asserts:
         constfold(config, graphs)
         remove_asserts(translator, graphs)
diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py
--- a/rpython/translator/backendopt/constfold.py
+++ b/rpython/translator/backendopt/constfold.py
@@ -276,3 +276,25 @@
             rewire_links(splitblocks, graph)
         if not diffused and not splitblocks:
             break # finished
+
+def replace_symbolic(graph, symbolic, value):
+    result = False
+    for block in graph.iterblocks():
+        for op in block.operations:
+            for i, arg in enumerate(op.args):
+                if isinstance(arg, Constant) and arg.value is symbolic:
+                    op.args[i] = value
+                    result = True
+        if block.exitswitch is symbolic:
+            block.exitswitch = value
+            result = True
+    return result
+
+def replace_we_are_jitted(graph):
+    from rpython.rlib import jit
+    replacement = Constant(0)
+    replacement.concretetype = lltype.Signed
+    did_replacement = replace_symbolic(graph, jit._we_are_jitted, replacement)
+    if did_replacement:
+        constant_fold_graph(graph)
+    return did_replacement
diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py
--- a/rpython/translator/backendopt/test/test_all.py
+++ b/rpython/translator/backendopt/test/test_all.py
@@ -289,3 +289,19 @@
         llinterp = LLInterpreter(t.rtyper)
         res = llinterp.eval_graph(later_graph, [10])
         assert res == 1
+
+    def test_replace_we_are_jitted(self):
+        from rpython.rlib import jit
+        def f():
+            if jit.we_are_jitted():
+                return 1
+            return 2 + jit.we_are_jitted()
+
+        t = self.translateopt(f, [])
+        graph = graphof(t, f)
+        # by default, replace_we_are_jitted is off
+        assert graph.startblock.operations[0].args[0].value is jit._we_are_jitted
+
+        t = self.translateopt(f, [], replace_we_are_jitted=True)
+        graph = graphof(t, f)
+        assert graph.startblock.exits[0].args[0].value == 2
diff --git a/rpython/translator/backendopt/test/test_constfold.py b/rpython/translator/backendopt/test/test_constfold.py
--- a/rpython/translator/backendopt/test/test_constfold.py
+++ b/rpython/translator/backendopt/test/test_constfold.py
@@ -7,6 +7,7 @@
 from rpython.rtyper import rclass
 from rpython.rlib import objectmodel
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.conftest import option
 
 def get_graph(fn, signature):
@@ -343,3 +344,18 @@
     merge_if_blocks.merge_if_blocks_once(graph)
     constant_fold_graph(graph)
     check_graph(graph, [], 66, t)
+
+def test_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def fn():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+    graph, t = get_graph(fn, [])
+    result = replace_we_are_jitted(graph)
+    assert result
+    checkgraph(graph)
+    # check shape of graph
+    assert len(graph.startblock.operations) == 0
+    assert graph.startblock.exitswitch is None
+    assert graph.startblock.exits[0].target.exits[0].args[0].value == 2
diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -381,7 +381,7 @@
         """ Run all backend optimizations - lltype version
         """
         from rpython.translator.backendopt.all import backend_optimizations
-        backend_optimizations(self.translator)
+        backend_optimizations(self.translator, replace_we_are_jitted=True)
 
 
     STACKCHECKINSERTION = 'stackcheckinsertion_lltype'
diff --git a/rpython/translator/test/test_interactive.py b/rpython/translator/test/test_interactive.py
--- a/rpython/translator/test/test_interactive.py
+++ b/rpython/translator/test/test_interactive.py
@@ -78,3 +78,15 @@
     dll = ctypes.CDLL(str(t.driver.c_entryp))
     f = dll.pypy_g_f
     assert f(2, 3) == 5
+
+def test_check_that_driver_uses_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def f():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+
+    t = Translation(f, [])
+    t.backendopt()
+    graph = t.driver.translator.graphs[0]
+    assert graph.startblock.exits[0].args[0].value == 2

From pypy.commits at gmail.com  Wed Aug 24 03:14:20 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 00:14:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge memoryview-attributes
Message-ID: <57bd494c.81a2c20a.cf2ff.5289@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r86456:0d29b975452d
Date: 2016-08-24 09:10 +0200
http://bitbucket.org/pypy/pypy/changeset/0d29b975452d/

Log:	merge memoryview-attributes

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
         else:
             return self.value
 
-    def __buffer__(self):
+    def __buffer__(self, flags):
         return buffer(self._buffer)
 
     def _get_b_base(self):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -208,7 +208,8 @@
     def buffer_w(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self, 
+                                        space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
@@ -216,7 +217,8 @@
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
@@ -224,7 +226,8 @@
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
@@ -232,7 +235,8 @@
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -121,7 +121,7 @@
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 """.split()
 for name in constant_names:
@@ -649,6 +649,7 @@
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
         ))
+Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,13 +1,17 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer)
+    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, w_obj):
+def PyObject_CheckBuffer(space, pyobj):
     """Return 1 if obj supports the buffer interface otherwise 0."""
-    return 0  # the bf_getbuffer field is never filled by cpyext
+    as_buffer = pyobj.c_ob_type.c_tp_as_buffer
+    flags = pyobj.c_ob_type.c_tp_flags
+    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
+        return 1
+    return 0  
 
 @cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
              rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -12,7 +12,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    raise NotImplementedError
+    raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
 def PyMemoryView_GET_BUFFER(space, w_obj):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -3,15 +3,16 @@
 import re
 
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl)
+    mangle_name, pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    readbufferproc, ssizessizeobjargproc)
+    readbufferproc, getbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -22,6 +23,9 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.sys.version import CPYTHON_VERSION
+
+PY3 = CPYTHON_VERSION[0] == 3
 
 # XXX: Also defined in object.h
 Py_LT = 0
@@ -298,11 +302,23 @@
     # Similar to Py_buffer
     _immutable_ = True
 
-    def __init__(self, ptr, size, w_obj):
+    def __init__(self, ptr, size, w_obj, format='B', shape=None,
+                strides=None, ndim=1, itemsize=1, readonly=True):
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
-        self.readonly = True
+        self.format = format
+        if not shape:
+            self.shape = [size]
+        else:
+            self.shape = shape
+        if not strides:
+            self.strides = [1]
+        else:
+            self.strides = strides
+        self.ndim = ndim 
+        self.itemsize = itemsize
+        self.readonly = readonly
 
     def getlength(self):
         return self.size
@@ -313,6 +329,15 @@
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
 
+    def getformat(self):
+        return self.format
+
+    def getshape(self):
+        return self.shape
+
+    def getitemsize(self):
+        return self.itemsize
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
@@ -322,6 +347,30 @@
             space.fromcache(State).check_and_raise_exception(always=True)
         return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
 
+def wrap_getbuffer(space, w_self, w_args, func):
+    func_target = rffi.cast(getbufferproc, func)
+    with lltype.scoped_alloc(Py_buffer) as pybuf:
+        _flags = 0
+        if space.len_w(w_args) > 0:
+            _flags = space.int_w(space.listview(w_args)[0])
+        flags = rffi.cast(rffi.INT_real,_flags)
+        size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+        if widen(size) < 0:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        ptr = pybuf.c_buf
+        size = pybuf.c_len
+        ndim = widen(pybuf.c_ndim)
+        shape =   [pybuf.c_shape[i]   for i in range(ndim)]
+        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_format:
+            format = rffi.charp2str(pybuf.c_format)
+        else:
+            format = 'B'
+        return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
+                            ndim=ndim, shape=shape, strides=strides,
+                            itemsize=pybuf.c_itemsize,
+                            readonly=widen(pybuf.c_readonly)))
+
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
         func_target = rffi.cast(richcmpfunc, func)
@@ -486,7 +535,6 @@
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
         api_func = slot_tp_getattro.api_func
-
     elif name == 'tp_call':
         call_fn = w_type.getdictvalue(space, '__call__')
         if call_fn is None:
@@ -542,6 +590,21 @@
                              w_stararg=w_args, w_starstararg=w_kwds)
             return space.call_args(space.get(new_fn, w_self), args)
         api_func = slot_tp_new.api_func
+    elif name == 'tp_as_buffer.c_bf_getbuffer':
+        buff_fn = w_type.getdictvalue(space, '__buffer__')
+        if buff_fn is None:
+            return
+        @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
+                rffi.INT_real, header=None, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def buff_w(space, w_self, pybuf, flags):
+            # XXX this is wrong, needs a test
+            raise oefmt(space.w_NotImplemented, 
+                "calling bf_getbuffer on a builtin type not supported yet")
+            #args = Arguments(space, [w_self],
+            #                 w_stararg=w_args, w_starstararg=w_kwds)
+            #return space.call_args(space.get(buff_fn, w_self), args)
+        api_func = buff_w.api_func
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
@@ -850,11 +913,19 @@
 slotdefs = eval(slotdefs_str)
 # PyPy addition
 slotdefs += (
-    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    # XXX that might not be what we want!
+    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""),
 )
 
+if not PY3:
+    slotdefs += (
+        TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    )
+
+
 # partial sort to solve some slot conflicts:
 # Number slots before Mapping slots before Sequence slots.
+# also prefer the new buffer interface
 # These are the only conflicts between __name__ methods
 def slotdef_sort_key(slotdef):
     if slotdef.slot_name.startswith('tp_as_number'):
@@ -863,6 +934,10 @@
         return 2
     if slotdef.slot_name.startswith('tp_as_sequence'):
         return 3
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getbuffer':
+        return 100
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer':
+        return 101
     return 0
 slotdefs = sorted(slotdefs, key=slotdef_sort_key)
 
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -0,0 +1,243 @@
+#include <Python.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+/* 
+ * Adapted from https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol,
+ * which is copyright Jake Vanderplas and released under the BSD license
+ */
+
+/* Structure defines a 1-dimensional strided array */
+typedef struct{
+    int* arr;
+    long length;
+} MyArray;
+
+/* initialize the array with integers 0...length */
+void initialize_MyArray(MyArray* a, long length){
+    int i;
+    a->length = length;
+    a->arr = (int*)malloc(length * sizeof(int));
+    for(i=0; i<length; i++){
+        a->arr[i] = i;
+    }
+}
+
+/* free the memory when finished */
+void deallocate_MyArray(MyArray* a){
+    free(a->arr);
+    a->arr = NULL;
+}
+
+/* tools to print the array */
+char* stringify(MyArray* a, int nmax){
+    char* output = (char*) malloc(nmax * 20);
+    int k, pos = sprintf(&output[0], "[");
+
+    for (k=0; k < a->length && k < nmax; k++){
+        pos += sprintf(&output[pos], " %d", a->arr[k]);
+    }
+    if(a->length > nmax)
+        pos += sprintf(&output[pos], "...");
+    sprintf(&output[pos], " ]");
+    return output;
+}
+
+void print_MyArray(MyArray* a, int nmax){
+    char* s = stringify(a, nmax);
+    printf("%s", s);
+    free(s);
+}
+
+/* This is where we define the PyMyArray object structure */
+typedef struct {
+    PyObject_HEAD
+    /* Type-specific fields go below. */
+    MyArray arr;
+} PyMyArray;
+
+
+/* This is the __init__ function, implemented in C */
+static int
+PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
+{
+    // init may have already been called
+    if (self->arr.arr != NULL) {
+        deallocate_MyArray(&self->arr);
+    }
+
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
+    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
+        return -1;
+
+    if (length < 0)
+        length = 0;
+
+    initialize_MyArray(&self->arr, length);
+
+    return 0;
+}
+
+
+/* this function is called when the object is deallocated */
+static void
+PyMyArray_dealloc(PyMyArray* self)
+{
+    deallocate_MyArray(&self->arr);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+
+/* This function returns the string representation of our object */
+static PyObject *
+PyMyArray_str(PyMyArray * self)
+{
+  char* s = stringify(&self->arr, 10);
+  PyObject* ret = PyUnicode_FromString(s);
+  free(s);
+  return ret;
+}
+
+/* Here is the buffer interface function */
+static int
+PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
+{
+  if (view == NULL) {
+    PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
+    return -1;
+  }
+  if (flags == 0) {
+    PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
+    return -1;
+  }
+
+  PyMyArray* self = (PyMyArray*)obj;
+  view->obj = (PyObject*)self;
+  view->buf = (void*)self->arr.arr;
+  view->len = self->arr.length * sizeof(int);
+  view->readonly = 0;
+  view->itemsize = sizeof(int);
+  view->format = "i";  // integer
+  view->ndim = 1;
+  view->shape = &self->arr.length;  // length-1 sequence of dimensions
+  view->strides = &view->itemsize;  // for the simple case we can do this
+  view->suboffsets = NULL;
+  view->internal = NULL;
+
+  Py_INCREF(self);  // need to increase the reference count
+  return 0;
+}
+
+static PyBufferProcs PyMyArray_as_buffer = {
+#if PY_MAJOR_VERSION < 3
+  (readbufferproc)0,
+  (writebufferproc)0,
+  (segcountproc)0,
+  (charbufferproc)0,
+#endif
+  (getbufferproc)PyMyArray_getbuffer,
+  (releasebufferproc)0,  // we do not require any special release function
+};
+
+
+/* Here is the type structure: we put the above functions in the appropriate place
+   in order to actually define the Python object type */
+static PyTypeObject PyMyArrayType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "pymyarray.PyMyArray",        /* tp_name */
+    sizeof(PyMyArray),            /* tp_basicsize */
+    0,                            /* tp_itemsize */
+    (destructor)PyMyArray_dealloc,/* tp_dealloc */
+    0,                            /* tp_print */
+    0,                            /* tp_getattr */
+    0,                            /* tp_setattr */
+    0,                            /* tp_reserved */
+    (reprfunc)PyMyArray_str,      /* tp_repr */
+    0,                            /* tp_as_number */
+    0,                            /* tp_as_sequence */
+    0,                            /* tp_as_mapping */
+    0,                            /* tp_hash  */
+    0,                            /* tp_call */
+    (reprfunc)PyMyArray_str,      /* tp_str */
+    0,                            /* tp_getattro */
+    0,                            /* tp_setattro */
+    &PyMyArray_as_buffer,         /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */
+    "PyMyArray object",           /* tp_doc */
+    0,                            /* tp_traverse */
+    0,                            /* tp_clear */
+    0,                            /* tp_richcompare */
+    0,                            /* tp_weaklistoffset */
+    0,                            /* tp_iter */
+    0,                            /* tp_iternext */
+    0,                            /* tp_methods */
+    0,                            /* tp_members */
+    0,                            /* tp_getset */
+    0,                            /* tp_base */
+    0,                            /* tp_dict */
+    0,                            /* tp_descr_get */
+    0,                            /* tp_descr_set */
+    0,                            /* tp_dictoffset */
+    (initproc)PyMyArray_init,     /* tp_init */
+};
+
+static PyMethodDef buffer_functions[] = {
+    {NULL,        NULL}    /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "buffer_test",
+    "Module Doc",
+    -1,
+    buffer_functions;
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+};
+#define INITERROR return NULL
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+PyInit_buffer_test(void)
+
+#else
+
+#define INITERROR return
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+initbuffer_test(void)
+#endif
+{
+#if PY_MAJOR_VERSION >= 3
+    PyObject *m= PyModule_Create(&moduledef);
+#else
+    PyObject *m= Py_InitModule("buffer_test", buffer_functions);
+#endif
+    if (m == NULL)
+        INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&PyMyArrayType) < 0)
+        INITERROR;
+    Py_INCREF(&PyMyArrayType);
+    PyModule_AddObject(m, "PyMyArray", (PyObject *)&PyMyArrayType);
+#if PY_MAJOR_VERSION >=3
+    return m;
+#endif
+}
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,5 +1,7 @@
 import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
@@ -8,10 +10,23 @@
             py.test.skip("unsupported before Python 2.7")
 
         w_hello = space.newbytes("hello")
+        assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
+        w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
+        assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
-    @pytest.mark.skipif(True, reason='write a test for this')
-    def test_get_base_and_get_buffer(self, space, api):
-        assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER
+
+class AppTestBufferProtocol(AppTestCpythonExtensionBase):
+    def test_buffer_protocol(self):
+        import struct
+        module = self.import_module(name='buffer_test')
+        arr = module.PyMyArray(10)
+        y = memoryview(arr)
+        assert y.format == 'i'
+        assert y.shape == (10,)
+        s = y[3]
+        assert len(s) == struct.calcsize('i')
+        assert s == struct.pack('i', 3)
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -17,7 +17,8 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr)
+    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -608,6 +609,7 @@
                 bf_getwritebuffer.api_func.get_wrapper(space))
     pto.c_tp_as_buffer = c_buf
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
+    pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def type_dealloc(space, obj):
@@ -774,6 +776,8 @@
             pto.c_tp_setattro = base.c_tp_setattro
         if not pto.c_tp_getattro:
             pto.c_tp_getattro = base.c_tp_getattro
+        if not pto.c_tp_as_buffer:
+            pto.c_tp_as_buffer = base.c_tp_as_buffer
     finally:
         Py_DecRef(space, base_pyo)
 
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -5,6 +5,7 @@
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
+from pypy.module.cpyext.api import Py_bufferP
 
 
 P, FT, PyO = Ptr, FuncType, PyObject
@@ -58,8 +59,7 @@
 writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
 segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
 charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
-## We don't support new buffer interface for now
-getbufferproc = rffi.VOIDP
+getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = rffi.VOIDP
 
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -19,7 +19,6 @@
     UserDelAction)
 from pypy.interpreter.pyframe import PyFrame
 
-
 class BogusBytecode(Exception):
     pass
 
@@ -383,6 +382,9 @@
         # XXX even the hacks have hacks
         if s == 'size': # used in _array() but never called by tests
             return IntObject(0)
+        if s == '__buffer__':
+            # descr___buffer__ does not exist on W_Root
+            return self.w_None
         return getattr(w_obj, 'descr_' + s)(self, *args)
 
     @specialize.arg(1)
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -704,3 +704,20 @@
     def get_raw_address(self):
         from rpython.rtyper.lltypesystem import rffi
         return rffi.ptradd(self.impl.storage, self.impl.start)
+
+    def getformat(self):
+        return self.impl.dtype.char
+
+    def getitemsize(self):
+        return self.impl.dtype.elsize
+
+    def getndim(self):
+        return len(self.impl.shape)
+
+    def getshape(self):
+        return self.impl.shape
+
+    def getstrides(self):
+        return self.impl.strides
+
+
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -468,7 +468,8 @@
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_buffer = space.getattr(w_buffer, space.wrap('__buffer__'))
+        w_buffer = space.call_method(w_buffer, '__buffer__', 
+                                    space.newint(space.BUF_FULL_RO))
         buf = _getbuffer(space, w_buffer)
 
     ts = buf.getlength()
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3626,13 +3626,35 @@
         assert str(exc.value) == "assignment destination is read-only"
 
         class A(object):
-            __buffer__ = 'abc'
+            def __buffer__(self, flags):
+                return 'abc'
 
         data = A()
         a = np.frombuffer(data, 'c')
         #assert a.base is data.__buffer__
         assert a.tostring() == 'abc'
 
+    def test_memoryview(self):
+        import numpy as np
+        import sys
+        if sys.version_info[:2] > (3, 2):
+            # In Python 3.3 the representation of empty shape, strides and sub-offsets
+            # is an empty tuple instead of None.
+            # http://docs.python.org/dev/whatsnew/3.3.html#api-changes
+            EMPTY = ()
+        else:
+            EMPTY = None
+        x = np.array([1, 2, 3, 4, 5], dtype='i')
+        y = memoryview('abc')
+        assert y.format == 'B'
+        y = memoryview(x)
+        assert y.format == 'i'
+        assert y.shape == (5,)
+        assert y.ndim == 1
+        assert y.strides == (4,)
+        assert y.suboffsets == EMPTY
+        assert y.itemsize == 4
+
     def test_fromstring(self):
         import sys
         from numpy import fromstring, dtype
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -434,7 +434,6 @@
         of the specified width. The string S is never truncated.
         """
 
-
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
@@ -464,6 +463,11 @@
         raise oefmt(space.w_TypeError,
                     "Cannot use string as modifiable buffer")
 
+    def descr_getbuffer(self, space, w_flags):
+        #from pypy.objspace.std.bufferobject import W_Buffer
+        #return W_Buffer(StringBuffer(self._value))
+        return self
+
     charbuf_w = str_w
 
     def listview_bytes(self):
@@ -925,6 +929,7 @@
     translate = interpindirect2app(W_AbstractBytesObject.descr_translate),
     upper = interpindirect2app(W_AbstractBytesObject.descr_upper),
     zfill = interpindirect2app(W_AbstractBytesObject.descr_zfill),
+    __buffer__ = interp2app(W_BytesObject.descr_getbuffer),
 
     format = interpindirect2app(W_BytesObject.descr_format),
     __format__ = interpindirect2app(W_BytesObject.descr__format__),
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -73,6 +73,15 @@
 
     def descr_getitem(self, space, w_index):
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        itemsize = self.buf.getitemsize()
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+            stop  = start + size
+            if step == 0:
+                step = 1
+            if stop > self.getlength():
+                raise oefmt(space.w_IndexError, 'index out of range')
         if step not in (0, 1):
             raise oefmt(space.w_NotImplementedError, "")
         if step == 0:  # index only
@@ -85,6 +94,15 @@
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        itemsize = self.buf.getitemsize()
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+            stop  = start + size
+            if step == 0:
+                step = 1
+            if stop > self.getlength():
+                raise oefmt(space.w_IndexError, 'index out of range')
         if step not in (0, 1):
             raise oefmt(space.w_NotImplementedError, "")
         value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
@@ -100,22 +118,22 @@
         return space.wrap(self.buf.getlength())
 
     def w_get_format(self, space):
-        return space.wrap("B")
+        return space.wrap(self.buf.getformat())
 
     def w_get_itemsize(self, space):
-        return space.wrap(1)
+        return space.wrap(self.buf.getitemsize())
 
     def w_get_ndim(self, space):
-        return space.wrap(1)
+        return space.wrap(self.buf.getndim())
 
     def w_is_readonly(self, space):
-        return space.wrap(self.buf.readonly)
+        return space.newbool(bool(self.buf.readonly))
 
     def w_get_shape(self, space):
-        return space.newtuple([space.wrap(self.getlength())])
+        return space.newtuple([space.wrap(x) for x in self.buf.getshape()])
 
     def w_get_strides(self, space):
-        return space.newtuple([space.wrap(1)])
+        return space.newtuple([space.wrap(x) for x in self.buf.getstrides()])
 
     def w_get_suboffsets(self, space):
         # I've never seen anyone filling this field
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -4,7 +4,7 @@
     def test_init(self):
         import sys
         class A(object):
-            def __buffer__(self):
+            def __buffer__(self, flags):
                 return buffer('123')
         if '__pypy__' not in sys.builtin_module_names:
             raises(TypeError, buffer, A())
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -107,6 +107,9 @@
     def newlist(self, iterable):
         return list(iterable)
 
+    def newbytes(self, obj):
+        return bytes(obj)
+
     def call_function(self, func, *args, **kwds):
         return func(*args, **kwds)
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -59,6 +59,20 @@
     def get_raw_address(self):
         raise ValueError("no raw buffer")
 
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
 
 class StringBuffer(Buffer):
     __slots__ = ['value']

From pypy.commits at gmail.com  Wed Aug 24 03:14:23 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 00:14:23 -0700 (PDT)
Subject: [pypy-commit] pypy memoryview-attributes: close branch
Message-ID: <57bd494f.87941c0a.7b791.7dc1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: memoryview-attributes
Changeset: r86458:9ac7d2defab0
Date: 2016-08-24 09:13 +0200
http://bitbucket.org/pypy/pypy/changeset/9ac7d2defab0/

Log:	close branch


From pypy.commits at gmail.com  Wed Aug 24 03:14:22 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 00:14:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <57bd494e.82cbc20a.a3db7.4c18@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r86457:951e70bd2fa1
Date: 2016-08-24 09:13 +0200
http://bitbucket.org/pypy/pypy/changeset/951e70bd2fa1/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -150,3 +150,8 @@
 
 Reduce the size of the generated C code by constant-folding ``we_are_jitted``
 in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.

From pypy.commits at gmail.com  Wed Aug 24 04:14:40 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 01:14:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <57bd5770.a719c20a.7a270.67a7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86459:281ace7115cb
Date: 2016-08-24 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/281ace7115cb/

Log:	merge default

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -166,8 +166,8 @@
         else:
             return self.value
 
-    def __buffer__(self):
-        return memoryview(self._buffer)
+    def __buffer__(self, flags):
+        return buffer(self._buffer)
 
     def _get_b_base(self):
         try:
@@ -208,7 +208,7 @@
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
-    address = address & (sys.maxsize * 2 + 1)
+    address = address & (sys.maxint * 2 + 1)
     instance = self.__new__(self)
     lgt = getattr(self, '_length_', 1)
     instance._buffer = self._ffiarray.fromaddress(address, lgt)
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -150,3 +150,8 @@
 
 Reduce the size of the generated C code by constant-folding ``we_are_jitted``
 in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -585,6 +585,11 @@
         self.sys = Module(self, w_name)
         self.sys.install()
 
+        from pypy.module.imp import Module
+        w_name = self.wrap('imp')
+        mod = Module(self, w_name)
+        mod.install()
+
         from pypy.module.__builtin__ import Module
         w_name = self.wrap('builtins')
         self.builtin = Module(self, w_name)
@@ -1993,7 +1998,7 @@
 
 ObjSpace.IrregularOpTable = [
     'wrap',
-    'bytes_w',
+    'str_w',
     'int_w',
     'float_w',
     'uint_w',
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,7 +120,7 @@
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 Py_CLEANUP_SUPPORTED
 """.split()
@@ -651,6 +651,7 @@
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
         ))
+Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,8 +1,37 @@
+from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import buffer
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer)
-from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
+from pypy.module.cpyext.pyobject import PyObject
+
+ at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
+def PyObject_CheckBuffer(space, w_obj):
+    """Return 1 if obj supports the buffer interface otherwise 0."""
+    return 0  # the bf_getbuffer field is never filled by cpyext
+
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+             rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+    """Export obj into a Py_buffer, view.  These arguments must
+    never be NULL.  The flags argument is a bit field indicating what
+    kind of buffer the caller is prepared to deal with and therefore what
+    kind of buffer the exporter is allowed to return.  The buffer interface
+    allows for complicated memory sharing possibilities, but some caller may
+    not be able to handle all the complexity but may want to see if the
+    exporter will let them take a simpler view to its memory.
+
+    Some exporters may not be able to share memory in every possible way and
+    may need to raise errors to signal to some consumers that something is
+    just not possible. These errors should be a BufferError unless
+    there is another error that is actually causing the problem. The
+    exporter can use flags information to simplify how much of the
+    Py_buffer structure is filled in with non-default values and/or
+    raise an error if the object can't support a simpler view of its memory.
+
+    0 is returned on success and -1 on error."""
+    raise oefmt(space.w_TypeError,
+                "PyPy does not yet implement the new buffer interface")
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
 def PyBuffer_IsContiguous(space, view, fortran):
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -16,7 +16,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    raise NotImplementedError
+    raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
 def PyMemoryView_GET_BUFFER(space, w_obj):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,11 +7,11 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
     Py_buffer, mangle_name, pypy_decl)
 from pypy.module.cpyext.typeobjectdefs import (
-    unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
+    unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    getbufferproc, ssizessizeobjargproc)
+    getbufferproc, readbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -298,11 +298,23 @@
     # Similar to Py_buffer
     _immutable_ = True
 
-    def __init__(self, ptr, size, w_obj):
+    def __init__(self, ptr, size, w_obj, format='B', shape=None,
+                strides=None, ndim=1, itemsize=1, readonly=True):
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
-        self.readonly = True
+        self.format = format
+        if not shape:
+            self.shape = [size]
+        else:
+            self.shape = shape
+        if not strides:
+            self.strides = [1]
+        else:
+            self.strides = strides
+        self.ndim = ndim 
+        self.itemsize = itemsize
+        self.readonly = readonly
 
     def getlength(self):
         return self.size
@@ -313,14 +325,38 @@
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
 
+    def getformat(self):
+        return self.format
+
+    def getshape(self):
+        return self.shape
+
+    def getitemsize(self):
+        return self.itemsize
+
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
-    with lltype.scoped_alloc(Py_buffer) as view:
-        flags = rffi.cast(rffi.INT_real, 0)
-        ret = generic_cpy_call(space, func_target, w_self, view, flags)
-        if rffi.cast(lltype.Signed, ret) == -1:
+    with lltype.scoped_alloc(Py_buffer) as pybuf:
+        _flags = 0
+        if space.len_w(w_args) > 0:
+            _flags = space.int_w(space.listview(w_args)[0])
+        flags = rffi.cast(rffi.INT_real,_flags)
+        size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+        if widen(size) < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        return space.newbuffer(CPyBuffer(view.c_buf, view.c_len, w_self))
+        ptr = pybuf.c_buf
+        size = pybuf.c_len
+        ndim = widen(pybuf.c_ndim)
+        shape =   [pybuf.c_shape[i]   for i in range(ndim)]
+        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_format:
+            format = rffi.charp2str(pybuf.c_format)
+        else:
+            format = 'B'
+        return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
+                            ndim=ndim, shape=shape, strides=strides,
+                            itemsize=pybuf.c_itemsize,
+                            readonly=widen(pybuf.c_readonly)))
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
@@ -542,6 +578,21 @@
                              w_stararg=w_args, w_starstararg=w_kwds)
             return space.call_args(space.get(new_fn, w_self), args)
         api_func = slot_tp_new.api_func
+    elif name == 'tp_as_buffer.c_bf_getbuffer':
+        buff_fn = w_type.getdictvalue(space, '__buffer__')
+        if buff_fn is None:
+            return
+        @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
+                rffi.INT_real, header=None, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def buff_w(space, w_self, pybuf, flags):
+            # XXX this is wrong, needs a test
+            raise oefmt(space.w_NotImplemented, 
+                "calling bf_getbuffer on a builtin type not supported yet")
+            #args = Arguments(space, [w_self],
+            #                 w_stararg=w_args, w_starstararg=w_kwds)
+            #return space.call_args(space.get(buff_fn, w_self), args)
+        api_func = buff_w.api_func
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
@@ -673,23 +724,23 @@
                "x.__delitem__(y) <==> del x[y]"),
 
         BINSLOT("__add__", nb_add, slot_nb_add,
-            "+"),
+                "+"),
         RBINSLOT("__radd__", nb_add, slot_nb_add,
                  "+"),
         BINSLOT("__sub__", nb_subtract, slot_nb_subtract,
-            "-"),
+                "-"),
         RBINSLOT("__rsub__", nb_subtract, slot_nb_subtract,
                  "-"),
         BINSLOT("__mul__", nb_multiply, slot_nb_multiply,
-            "*"),
+                "*"),
         RBINSLOT("__rmul__", nb_multiply, slot_nb_multiply,
                  "*"),
         BINSLOT("__mod__", nb_remainder, slot_nb_remainder,
-            "%"),
+                "%"),
         RBINSLOT("__rmod__", nb_remainder, slot_nb_remainder,
                  "%"),
         BINSLOTNOTINFIX("__divmod__", nb_divmod, slot_nb_divmod,
-            "divmod(x, y)"),
+                        "divmod(x, y)"),
         RBINSLOTNOTINFIX("__rdivmod__", nb_divmod, slot_nb_divmod,
                  "divmod(y, x)"),
         NBSLOT("__pow__", nb_power, slot_nb_power, wrap_ternaryfunc,
@@ -819,11 +870,19 @@
 slotdefs = eval(slotdefs_str)
 # PyPy addition
 slotdefs += (
+    # XXX that might not be what we want!
     TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""),
 )
 
+if not PY3:
+    slotdefs += (
+        TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    )
+
+
 # partial sort to solve some slot conflicts:
 # Number slots before Mapping slots before Sequence slots.
+# also prefer the new buffer interface
 # These are the only conflicts between __name__ methods
 def slotdef_sort_key(slotdef):
     if slotdef.slot_name.startswith('tp_as_number'):
@@ -832,6 +891,10 @@
         return 2
     if slotdef.slot_name.startswith('tp_as_sequence'):
         return 3
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getbuffer':
+        return 100
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer':
+        return 101
     return 0
 slotdefs = sorted(slotdefs, key=slotdef_sort_key)
 
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -0,0 +1,243 @@
+#include <Python.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+/* 
+ * Adapted from https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol,
+ * which is copyright Jake Vanderplas and released under the BSD license
+ */
+
+/* Structure defines a 1-dimensional strided array */
+typedef struct{
+    int* arr;
+    long length;
+} MyArray;
+
+/* initialize the array with integers 0...length */
+void initialize_MyArray(MyArray* a, long length){
+    int i;
+    a->length = length;
+    a->arr = (int*)malloc(length * sizeof(int));
+    for(i=0; i<length; i++){
+        a->arr[i] = i;
+    }
+}
+
+/* free the memory when finished */
+void deallocate_MyArray(MyArray* a){
+    free(a->arr);
+    a->arr = NULL;
+}
+
+/* tools to print the array */
+char* stringify(MyArray* a, int nmax){
+    char* output = (char*) malloc(nmax * 20);
+    int k, pos = sprintf(&output[0], "[");
+
+    for (k=0; k < a->length && k < nmax; k++){
+        pos += sprintf(&output[pos], " %d", a->arr[k]);
+    }
+    if(a->length > nmax)
+        pos += sprintf(&output[pos], "...");
+    sprintf(&output[pos], " ]");
+    return output;
+}
+
+void print_MyArray(MyArray* a, int nmax){
+    char* s = stringify(a, nmax);
+    printf("%s", s);
+    free(s);
+}
+
+/* This is where we define the PyMyArray object structure */
+typedef struct {
+    PyObject_HEAD
+    /* Type-specific fields go below. */
+    MyArray arr;
+} PyMyArray;
+
+
+/* This is the __init__ function, implemented in C */
+static int
+PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
+{
+    // init may have already been called
+    if (self->arr.arr != NULL) {
+        deallocate_MyArray(&self->arr);
+    }
+
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
+    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
+        return -1;
+
+    if (length < 0)
+        length = 0;
+
+    initialize_MyArray(&self->arr, length);
+
+    return 0;
+}
+
+
+/* this function is called when the object is deallocated */
+static void
+PyMyArray_dealloc(PyMyArray* self)
+{
+    deallocate_MyArray(&self->arr);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+
+/* This function returns the string representation of our object */
+static PyObject *
+PyMyArray_str(PyMyArray * self)
+{
+  char* s = stringify(&self->arr, 10);
+  PyObject* ret = PyUnicode_FromString(s);
+  free(s);
+  return ret;
+}
+
+/* Here is the buffer interface function */
+static int
+PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
+{
+  if (view == NULL) {
+    PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
+    return -1;
+  }
+  if (flags == 0) {
+    PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
+    return -1;
+  }
+
+  PyMyArray* self = (PyMyArray*)obj;
+  view->obj = (PyObject*)self;
+  view->buf = (void*)self->arr.arr;
+  view->len = self->arr.length * sizeof(int);
+  view->readonly = 0;
+  view->itemsize = sizeof(int);
+  view->format = "i";  // integer
+  view->ndim = 1;
+  view->shape = &self->arr.length;  // length-1 sequence of dimensions
+  view->strides = &view->itemsize;  // for the simple case we can do this
+  view->suboffsets = NULL;
+  view->internal = NULL;
+
+  Py_INCREF(self);  // need to increase the reference count
+  return 0;
+}
+
+static PyBufferProcs PyMyArray_as_buffer = {
+#if PY_MAJOR_VERSION < 3
+  (readbufferproc)0,
+  (writebufferproc)0,
+  (segcountproc)0,
+  (charbufferproc)0,
+#endif
+  (getbufferproc)PyMyArray_getbuffer,
+  (releasebufferproc)0,  // we do not require any special release function
+};
+
+
+/* Here is the type structure: we put the above functions in the appropriate place
+   in order to actually define the Python object type */
+static PyTypeObject PyMyArrayType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "pymyarray.PyMyArray",        /* tp_name */
+    sizeof(PyMyArray),            /* tp_basicsize */
+    0,                            /* tp_itemsize */
+    (destructor)PyMyArray_dealloc,/* tp_dealloc */
+    0,                            /* tp_print */
+    0,                            /* tp_getattr */
+    0,                            /* tp_setattr */
+    0,                            /* tp_reserved */
+    (reprfunc)PyMyArray_str,      /* tp_repr */
+    0,                            /* tp_as_number */
+    0,                            /* tp_as_sequence */
+    0,                            /* tp_as_mapping */
+    0,                            /* tp_hash  */
+    0,                            /* tp_call */
+    (reprfunc)PyMyArray_str,      /* tp_str */
+    0,                            /* tp_getattro */
+    0,                            /* tp_setattro */
+    &PyMyArray_as_buffer,         /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */
+    "PyMyArray object",           /* tp_doc */
+    0,                            /* tp_traverse */
+    0,                            /* tp_clear */
+    0,                            /* tp_richcompare */
+    0,                            /* tp_weaklistoffset */
+    0,                            /* tp_iter */
+    0,                            /* tp_iternext */
+    0,                            /* tp_methods */
+    0,                            /* tp_members */
+    0,                            /* tp_getset */
+    0,                            /* tp_base */
+    0,                            /* tp_dict */
+    0,                            /* tp_descr_get */
+    0,                            /* tp_descr_set */
+    0,                            /* tp_dictoffset */
+    (initproc)PyMyArray_init,     /* tp_init */
+};
+
+static PyMethodDef buffer_functions[] = {
+    {NULL,        NULL}    /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "buffer_test",
+    "Module Doc",
+    -1,
+    buffer_functions;
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+};
+#define INITERROR return NULL
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+PyInit_buffer_test(void)
+
+#else
+
+#define INITERROR return
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+initbuffer_test(void)
+#endif
+{
+#if PY_MAJOR_VERSION >= 3
+    PyObject *m= PyModule_Create(&moduledef);
+#else
+    PyObject *m= Py_InitModule("buffer_test", buffer_functions);
+#endif
+    if (m == NULL)
+        INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&PyMyArrayType) < 0)
+        INITERROR;
+    Py_INCREF(&PyMyArrayType);
+    PyModule_AddObject(m, "PyMyArray", (PyObject *)&PyMyArrayType);
+#if PY_MAJOR_VERSION >=3
+    return m;
+#endif
+}
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -9,7 +9,10 @@
             py.test.skip("unsupported before Python 2.7")
 
         w_hello = space.newbytes("hello")
+        assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
+        w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
+        assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
@@ -55,3 +58,15 @@
     @pytest.mark.skipif(True, reason='write a test for this')
     def test_get_base_and_get_buffer(self, space, api):
         assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER
+
+class AppTestBufferProtocol(AppTestCpythonExtensionBase):
+    def test_buffer_protocol(self):
+        import struct
+        module = self.import_module(name='buffer_test')
+        arr = module.PyMyArray(10)
+        y = memoryview(arr)
+        assert y.format == 'i'
+        assert y.shape == (10,)
+        s = y[3]
+        assert len(s) == struct.calcsize('i')
+        assert s == struct.pack('i', 3)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -17,7 +17,8 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, Py_buffer, PyTypeObject, PyTypeObjectPtr)
+    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -514,6 +515,7 @@
         bytes_getbuffer.api_func.get_wrapper(space))
     pto.c_tp_as_buffer = c_buf
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
+    pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def type_dealloc(space, obj):
@@ -681,6 +683,8 @@
             pto.c_tp_setattro = base.c_tp_setattro
         if not pto.c_tp_getattro:
             pto.c_tp_getattro = base.c_tp_getattro
+        if not pto.c_tp_as_buffer:
+            pto.c_tp_as_buffer = base.c_tp_as_buffer
     finally:
         Py_DecRef(space, base_pyo)
 
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -1,10 +1,11 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.lltypesystem.lltype import Ptr, FuncType, Void
 from pypy.module.cpyext.api import (cpython_struct, Py_ssize_t, Py_ssize_tP,
-    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_buffer,
+    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP,
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
+from pypy.module.cpyext.api import Py_bufferP
 
 
 P, FT, PyO = Ptr, FuncType, PyObject
@@ -53,7 +54,11 @@
 wrapperfunc = P(FT([PyO, PyO, rffi.VOIDP], PyO))
 wrapperfunc_kwds = P(FT([PyO, PyO, rffi.VOIDP, PyO], PyO))
 
-getbufferproc = P(FT([PyO, Ptr(Py_buffer), rffi.INT_real], rffi.INT_real))
+readbufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
+writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
+segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
+charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
+getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = P(FT([PyO, Ptr(Py_buffer)], Void))
 
 
@@ -126,6 +131,10 @@
 ))
 
 PyBufferProcs = cpython_struct("PyBufferProcs", (
+    ("bf_getreadbuffer", readbufferproc),
+    ("bf_getwritebuffer", writebufferproc),
+    ("bf_getsegcount", segcountproc),
+    ("bf_getcharbuffer", charbufferproc),
     ("bf_getbuffer", getbufferproc),
     ("bf_releasebuffer", releasebufferproc),
 ))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -19,7 +19,6 @@
     UserDelAction)
 from pypy.interpreter.pyframe import PyFrame
 
-
 class BogusBytecode(Exception):
     pass
 
@@ -383,6 +382,9 @@
         # XXX even the hacks have hacks
         if s == 'size': # used in _array() but never called by tests
             return IntObject(0)
+        if s == '__buffer__':
+            # descr___buffer__ does not exist on W_Root
+            return self.w_None
         return getattr(w_obj, 'descr_' + s)(self, *args)
 
     @specialize.arg(1)
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -704,3 +704,20 @@
     def get_raw_address(self):
         from rpython.rtyper.lltypesystem import rffi
         return rffi.ptradd(self.impl.storage, self.impl.start)
+
+    def getformat(self):
+        return self.impl.dtype.char
+
+    def getitemsize(self):
+        return self.impl.dtype.elsize
+
+    def getndim(self):
+        return len(self.impl.shape)
+
+    def getshape(self):
+        return self.impl.shape
+
+    def getstrides(self):
+        return self.impl.strides
+
+
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -468,7 +468,8 @@
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_buffer = space.getattr(w_buffer, space.wrap('__buffer__'))
+        w_buffer = space.call_method(w_buffer, '__buffer__', 
+                                    space.newint(space.BUF_FULL_RO))
         buf = _getbuffer(space, w_buffer)
 
     ts = buf.getlength()
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3617,13 +3617,35 @@
         assert str(exc.value) == "assignment destination is read-only"
 
         class A(object):
-            __buffer__ = 'abc'
+            def __buffer__(self, flags):
+                return 'abc'
 
         data = A()
         a = np.frombuffer(data, 'c')
         #assert a.base is data.__buffer__
         assert a.tostring() == 'abc'
 
+    def test_memoryview(self):
+        import numpy as np
+        import sys
+        if sys.version_info[:2] > (3, 2):
+            # In Python 3.3 the representation of empty shape, strides and sub-offsets
+            # is an empty tuple instead of None.
+            # http://docs.python.org/dev/whatsnew/3.3.html#api-changes
+            EMPTY = ()
+        else:
+            EMPTY = None
+        x = np.array([1, 2, 3, 4, 5], dtype='i')
+        y = memoryview('abc')
+        assert y.format == 'B'
+        y = memoryview(x)
+        assert y.format == 'i'
+        assert y.shape == (5,)
+        assert y.ndim == 1
+        assert y.strides == (4,)
+        assert y.suboffsets == EMPTY
+        assert y.itemsize == 4
+
     def test_fromstring(self):
         import sys
         from numpy import fromstring, dtype
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -12,7 +12,13 @@
 from pypy.interpreter.gateway import (
     WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
 from pypy.interpreter.typedef import TypeDef
+from pypy.objspace.std import newformat
+from pypy.objspace.std.basestringtype import basestring_typedef
+from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringmethods import StringMethods
+from pypy.objspace.std.unicodeobject import (
+    decode_object, unicode_from_encoded_object,
+    unicode_from_string, getdefaultencoding)
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 
@@ -394,7 +400,6 @@
         of the specified width. The string S is never truncated.
         """
 
-
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
@@ -417,6 +422,18 @@
         space.check_buf_flags(flags, True)
         return StringBuffer(self._value)
 
+    def readbuf_w(self, space):
+        return StringBuffer(self._value)
+
+    def writebuf_w(self, space):
+        raise oefmt(space.w_TypeError,
+                    "Cannot use string as modifiable buffer")
+
+    def descr_getbuffer(self, space, w_flags):
+        #from pypy.objspace.std.bufferobject import W_Buffer
+        #return W_Buffer(StringBuffer(self._value))
+        return self
+
     def listview_int(self):
         return _create_list_from_bytes(self._value)
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -82,18 +82,29 @@
     def descr_getitem(self, space, w_index):
         self._check_released(space)
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        itemsize = self.buf.getitemsize()
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+            stop  = start + size
+            if step == 0:
+                step = 1
+            if stop > self.getlength():
+                raise oefmt(space.w_IndexError, 'index out of range')
+        if step not in (0, 1):
+            raise oefmt(space.w_NotImplementedError, "")
         if step == 0:  # index only
             # TODO: this probably isn't very fast
-            buf = SubBuffer(self.buf, start * self.itemsize, self.itemsize)
+            buf = SubBuffer(self.buf, start, self.itemsize)
             fmtiter = UnpackFormatIterator(space, buf)
             fmtiter.interpret(self.format)
             return fmtiter.result_w[0]
         elif step == 1:
-            buf = SubBuffer(self.buf, start * self.itemsize,
-                            size * self.itemsize)
+            buf = SubBuffer(self.buf, start, size)
             return W_MemoryView(buf, self.format, self.itemsize)
         else:
-            raise oefmt(space.w_NotImplementedError, "")
+            buf = SubBuffer(self.buf, start, size)
+            return W_MemoryView(buf)
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
@@ -102,6 +113,21 @@
         if space.isinstance_w(w_index, space.w_tuple):
             raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        itemsize = self.buf.getitemsize()
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+            stop  = start + size
+            if step == 0:
+                step = 1
+            if stop > self.getlength():
+                raise oefmt(space.w_IndexError, 'index out of range')
+        if step not in (0, 1):
+            raise oefmt(space.w_NotImplementedError, "")
+        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+        if value.getlength() != size:
+            raise oefmt(space.w_ValueError,
+                        "cannot modify size of memoryview object")
         if step == 0:  # index only
             # TODO: this probably isn't very fast
             fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)
@@ -111,43 +137,43 @@
                 raise oefmt(space.w_TypeError,
                             "memoryview: invalid type for format '%s'",
                             self.format)
-            self.buf.setslice(start * self.itemsize, fmtiter.result.build())
+            self.buf.setslice(start, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
             if value.getlength() != size * self.itemsize:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
-            self.buf.setslice(start * self.itemsize, value.as_str())
+            self.buf.setslice(start, value.as_str())
         else:
             raise oefmt(space.w_NotImplementedError, "")
 
     def descr_len(self, space):
         self._check_released(space)
-        return space.wrap(self.getlength())
+        return space.wrap(self.buf.getlength())
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap(self.format)
+        return space.wrap(self.buf.getformat())
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.wrap(self.itemsize)
+        return space.wrap(self.buf.getitemsize())
 
     def w_get_ndim(self, space):
         self._check_released(space)
-        return space.wrap(1)
+        return space.wrap(self.buf.getndim())
 
     def w_is_readonly(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.readonly)
+        return space.newbool(bool(self.buf.readonly))
 
     def w_get_shape(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(self.getlength())])
+        return space.newtuple([space.wrap(x) for x in self.buf.getshape()])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(self.itemsize)])
+        return space.newtuple([space.wrap(x) for x in self.buf.getstrides()])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)
@@ -196,7 +222,6 @@
             raise OperationError(space.w_ValueError, space.wrap(msg))
         return space.wrap(rffi.cast(lltype.Signed, ptr))
 
-
 W_MemoryView.typedef = TypeDef(
     "memoryview",
     __doc__ = """\
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -122,6 +122,9 @@
     def newlist(self, iterable):
         return list(iterable)
 
+    def newbytes(self, obj):
+        return bytes(obj)
+
     def call_function(self, func, *args, **kwds):
         return func(*args, **kwds)
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -59,6 +59,20 @@
     def get_raw_address(self):
         raise ValueError("no raw buffer")
 
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
 
 class StringBuffer(Buffer):
     __slots__ = ['value']

From pypy.commits at gmail.com  Wed Aug 24 05:00:44 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 02:00:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge py3k
Message-ID: <57bd623c.81a2c20a.cf2ff.7d0e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86460:304a36465049
Date: 2016-08-24 10:29 +0200
http://bitbucket.org/pypy/pypy/changeset/304a36465049/

Log:	merge py3k

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -166,8 +166,8 @@
         else:
             return self.value
 
-    def __buffer__(self):
-        return memoryview(self._buffer)
+    def __buffer__(self, flags):
+        return buffer(self._buffer)
 
     def _get_b_base(self):
         try:
@@ -208,7 +208,7 @@
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
-    address = address & (sys.maxsize * 2 + 1)
+    address = address & (sys.maxint * 2 + 1)
     instance = self.__new__(self)
     lgt = getattr(self, '_length_', 1)
     instance._buffer = self._ffiarray.fromaddress(address, lgt)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -144,3 +144,14 @@
 ``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
 Previously it returned what looked like a regular dict object (but it
 was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -585,6 +585,11 @@
         self.sys = Module(self, w_name)
         self.sys.install()
 
+        from pypy.module.imp import Module
+        w_name = self.wrap('imp')
+        mod = Module(self, w_name)
+        mod.install()
+
         from pypy.module.__builtin__ import Module
         w_name = self.wrap('builtins')
         self.builtin = Module(self, w_name)
@@ -1996,7 +2001,7 @@
 
 ObjSpace.IrregularOpTable = [
     'wrap',
-    'bytes_w',
+    'str_w',
     'int_w',
     'float_w',
     'uint_w',
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -79,7 +79,7 @@
 
 
 class AppTestRecompiler:
-    spaceconfig = dict(usemodules=['_cffi_backend', 'imp', 'cpyext', 'struct'])
+    spaceconfig = dict(usemodules=['_cffi_backend', 'imp'])
 
     def setup_class(cls):
         if cls.runappdirect:
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,7 +120,7 @@
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 Py_CLEANUP_SUPPORTED
 """.split()
@@ -651,6 +651,7 @@
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
         ))
+Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,8 +1,37 @@
+from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import buffer
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer)
-from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
+from pypy.module.cpyext.pyobject import PyObject
+
+ at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
+def PyObject_CheckBuffer(space, w_obj):
+    """Return 1 if obj supports the buffer interface otherwise 0."""
+    return 0  # the bf_getbuffer field is never filled by cpyext
+
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+             rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+    """Export obj into a Py_buffer, view.  These arguments must
+    never be NULL.  The flags argument is a bit field indicating what
+    kind of buffer the caller is prepared to deal with and therefore what
+    kind of buffer the exporter is allowed to return.  The buffer interface
+    allows for complicated memory sharing possibilities, but some caller may
+    not be able to handle all the complexity but may want to see if the
+    exporter will let them take a simpler view to its memory.
+
+    Some exporters may not be able to share memory in every possible way and
+    may need to raise errors to signal to some consumers that something is
+    just not possible. These errors should be a BufferError unless
+    there is another error that is actually causing the problem. The
+    exporter can use flags information to simplify how much of the
+    Py_buffer structure is filled in with non-default values and/or
+    raise an error if the object can't support a simpler view of its memory.
+
+    0 is returned on success and -1 on error."""
+    raise oefmt(space.w_TypeError,
+                "PyPy does not yet implement the new buffer interface")
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
 def PyBuffer_IsContiguous(space, view, fortran):
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -16,7 +16,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    raise NotImplementedError
+    raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
 def PyMemoryView_GET_BUFFER(space, w_obj):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,11 +7,11 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
     Py_buffer, mangle_name, pypy_decl)
 from pypy.module.cpyext.typeobjectdefs import (
-    unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
+    unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    getbufferproc, ssizessizeobjargproc)
+    getbufferproc, readbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -298,11 +298,23 @@
     # Similar to Py_buffer
     _immutable_ = True
 
-    def __init__(self, ptr, size, w_obj):
+    def __init__(self, ptr, size, w_obj, format='B', shape=None,
+                strides=None, ndim=1, itemsize=1, readonly=True):
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
-        self.readonly = True
+        self.format = format
+        if not shape:
+            self.shape = [size]
+        else:
+            self.shape = shape
+        if not strides:
+            self.strides = [1]
+        else:
+            self.strides = strides
+        self.ndim = ndim 
+        self.itemsize = itemsize
+        self.readonly = readonly
 
     def getlength(self):
         return self.size
@@ -313,14 +325,38 @@
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
 
+    def getformat(self):
+        return self.format
+
+    def getshape(self):
+        return self.shape
+
+    def getitemsize(self):
+        return self.itemsize
+
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
-    with lltype.scoped_alloc(Py_buffer) as view:
-        flags = rffi.cast(rffi.INT_real, 0)
-        ret = generic_cpy_call(space, func_target, w_self, view, flags)
-        if rffi.cast(lltype.Signed, ret) == -1:
+    with lltype.scoped_alloc(Py_buffer) as pybuf:
+        _flags = 0
+        if space.len_w(w_args) > 0:
+            _flags = space.int_w(space.listview(w_args)[0])
+        flags = rffi.cast(rffi.INT_real,_flags)
+        size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+        if widen(size) < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        return space.newbuffer(CPyBuffer(view.c_buf, view.c_len, w_self))
+        ptr = pybuf.c_buf
+        size = pybuf.c_len
+        ndim = widen(pybuf.c_ndim)
+        shape =   [pybuf.c_shape[i]   for i in range(ndim)]
+        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_format:
+            format = rffi.charp2str(pybuf.c_format)
+        else:
+            format = 'B'
+        return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
+                            ndim=ndim, shape=shape, strides=strides,
+                            itemsize=pybuf.c_itemsize,
+                            readonly=widen(pybuf.c_readonly)))
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
@@ -542,6 +578,21 @@
                              w_stararg=w_args, w_starstararg=w_kwds)
             return space.call_args(space.get(new_fn, w_self), args)
         api_func = slot_tp_new.api_func
+    elif name == 'tp_as_buffer.c_bf_getbuffer':
+        buff_fn = w_type.getdictvalue(space, '__buffer__')
+        if buff_fn is None:
+            return
+        @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
+                rffi.INT_real, header=None, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def buff_w(space, w_self, pybuf, flags):
+            # XXX this is wrong, needs a test
+            raise oefmt(space.w_NotImplemented, 
+                "calling bf_getbuffer on a builtin type not supported yet")
+            #args = Arguments(space, [w_self],
+            #                 w_stararg=w_args, w_starstararg=w_kwds)
+            #return space.call_args(space.get(buff_fn, w_self), args)
+        api_func = buff_w.api_func
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
@@ -673,23 +724,23 @@
                "x.__delitem__(y) <==> del x[y]"),
 
         BINSLOT("__add__", nb_add, slot_nb_add,
-            "+"),
+                "+"),
         RBINSLOT("__radd__", nb_add, slot_nb_add,
                  "+"),
         BINSLOT("__sub__", nb_subtract, slot_nb_subtract,
-            "-"),
+                "-"),
         RBINSLOT("__rsub__", nb_subtract, slot_nb_subtract,
                  "-"),
         BINSLOT("__mul__", nb_multiply, slot_nb_multiply,
-            "*"),
+                "*"),
         RBINSLOT("__rmul__", nb_multiply, slot_nb_multiply,
                  "*"),
         BINSLOT("__mod__", nb_remainder, slot_nb_remainder,
-            "%"),
+                "%"),
         RBINSLOT("__rmod__", nb_remainder, slot_nb_remainder,
                  "%"),
         BINSLOTNOTINFIX("__divmod__", nb_divmod, slot_nb_divmod,
-            "divmod(x, y)"),
+                        "divmod(x, y)"),
         RBINSLOTNOTINFIX("__rdivmod__", nb_divmod, slot_nb_divmod,
                  "divmod(y, x)"),
         NBSLOT("__pow__", nb_power, slot_nb_power, wrap_ternaryfunc,
@@ -819,11 +870,19 @@
 slotdefs = eval(slotdefs_str)
 # PyPy addition
 slotdefs += (
+    # XXX that might not be what we want!
     TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""),
 )
 
+if not PY3:
+    slotdefs += (
+        TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    )
+
+
 # partial sort to solve some slot conflicts:
 # Number slots before Mapping slots before Sequence slots.
+# also prefer the new buffer interface
 # These are the only conflicts between __name__ methods
 def slotdef_sort_key(slotdef):
     if slotdef.slot_name.startswith('tp_as_number'):
@@ -832,6 +891,10 @@
         return 2
     if slotdef.slot_name.startswith('tp_as_sequence'):
         return 3
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getbuffer':
+        return 100
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer':
+        return 101
     return 0
 slotdefs = sorted(slotdefs, key=slotdef_sort_key)
 
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -0,0 +1,243 @@
+#include <Python.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+/* 
+ * Adapted from https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol,
+ * which is copyright Jake Vanderplas and released under the BSD license
+ */
+
+/* Structure defines a 1-dimensional strided array */
+typedef struct{
+    int* arr;
+    long length;
+} MyArray;
+
+/* initialize the array with integers 0...length */
+void initialize_MyArray(MyArray* a, long length){
+    int i;
+    a->length = length;
+    a->arr = (int*)malloc(length * sizeof(int));
+    for(i=0; i<length; i++){
+        a->arr[i] = i;
+    }
+}
+
+/* free the memory when finished */
+void deallocate_MyArray(MyArray* a){
+    free(a->arr);
+    a->arr = NULL;
+}
+
+/* tools to print the array */
+char* stringify(MyArray* a, int nmax){
+    char* output = (char*) malloc(nmax * 20);
+    int k, pos = sprintf(&output[0], "[");
+
+    for (k=0; k < a->length && k < nmax; k++){
+        pos += sprintf(&output[pos], " %d", a->arr[k]);
+    }
+    if(a->length > nmax)
+        pos += sprintf(&output[pos], "...");
+    sprintf(&output[pos], " ]");
+    return output;
+}
+
+void print_MyArray(MyArray* a, int nmax){
+    char* s = stringify(a, nmax);
+    printf("%s", s);
+    free(s);
+}
+
+/* This is where we define the PyMyArray object structure */
+typedef struct {
+    PyObject_HEAD
+    /* Type-specific fields go below. */
+    MyArray arr;
+} PyMyArray;
+
+
+/* This is the __init__ function, implemented in C */
+static int
+PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
+{
+    // init may have already been called
+    if (self->arr.arr != NULL) {
+        deallocate_MyArray(&self->arr);
+    }
+
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
+    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
+        return -1;
+
+    if (length < 0)
+        length = 0;
+
+    initialize_MyArray(&self->arr, length);
+
+    return 0;
+}
+
+
+/* this function is called when the object is deallocated */
+static void
+PyMyArray_dealloc(PyMyArray* self)
+{
+    deallocate_MyArray(&self->arr);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+
+/* This function returns the string representation of our object */
+static PyObject *
+PyMyArray_str(PyMyArray * self)
+{
+  char* s = stringify(&self->arr, 10);
+  PyObject* ret = PyUnicode_FromString(s);
+  free(s);
+  return ret;
+}
+
+/* Here is the buffer interface function */
+static int
+PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
+{
+  if (view == NULL) {
+    PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
+    return -1;
+  }
+  if (flags == 0) {
+    PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
+    return -1;
+  }
+
+  PyMyArray* self = (PyMyArray*)obj;
+  view->obj = (PyObject*)self;
+  view->buf = (void*)self->arr.arr;
+  view->len = self->arr.length * sizeof(int);
+  view->readonly = 0;
+  view->itemsize = sizeof(int);
+  view->format = "i";  // integer
+  view->ndim = 1;
+  view->shape = &self->arr.length;  // length-1 sequence of dimensions
+  view->strides = &view->itemsize;  // for the simple case we can do this
+  view->suboffsets = NULL;
+  view->internal = NULL;
+
+  Py_INCREF(self);  // need to increase the reference count
+  return 0;
+}
+
+static PyBufferProcs PyMyArray_as_buffer = {
+#if PY_MAJOR_VERSION < 3
+  (readbufferproc)0,
+  (writebufferproc)0,
+  (segcountproc)0,
+  (charbufferproc)0,
+#endif
+  (getbufferproc)PyMyArray_getbuffer,
+  (releasebufferproc)0,  // we do not require any special release function
+};
+
+
+/* Here is the type structure: we put the above functions in the appropriate place
+   in order to actually define the Python object type */
+static PyTypeObject PyMyArrayType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "pymyarray.PyMyArray",        /* tp_name */
+    sizeof(PyMyArray),            /* tp_basicsize */
+    0,                            /* tp_itemsize */
+    (destructor)PyMyArray_dealloc,/* tp_dealloc */
+    0,                            /* tp_print */
+    0,                            /* tp_getattr */
+    0,                            /* tp_setattr */
+    0,                            /* tp_reserved */
+    (reprfunc)PyMyArray_str,      /* tp_repr */
+    0,                            /* tp_as_number */
+    0,                            /* tp_as_sequence */
+    0,                            /* tp_as_mapping */
+    0,                            /* tp_hash  */
+    0,                            /* tp_call */
+    (reprfunc)PyMyArray_str,      /* tp_str */
+    0,                            /* tp_getattro */
+    0,                            /* tp_setattro */
+    &PyMyArray_as_buffer,         /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */
+    "PyMyArray object",           /* tp_doc */
+    0,                            /* tp_traverse */
+    0,                            /* tp_clear */
+    0,                            /* tp_richcompare */
+    0,                            /* tp_weaklistoffset */
+    0,                            /* tp_iter */
+    0,                            /* tp_iternext */
+    0,                            /* tp_methods */
+    0,                            /* tp_members */
+    0,                            /* tp_getset */
+    0,                            /* tp_base */
+    0,                            /* tp_dict */
+    0,                            /* tp_descr_get */
+    0,                            /* tp_descr_set */
+    0,                            /* tp_dictoffset */
+    (initproc)PyMyArray_init,     /* tp_init */
+};
+
+static PyMethodDef buffer_functions[] = {
+    {NULL,        NULL}    /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "buffer_test",
+    "Module Doc",
+    -1,
+    buffer_functions;
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+};
+#define INITERROR return NULL
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+PyInit_buffer_test(void)
+
+#else
+
+#define INITERROR return
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+initbuffer_test(void)
+#endif
+{
+#if PY_MAJOR_VERSION >= 3
+    PyObject *m= PyModule_Create(&moduledef);
+#else
+    PyObject *m= Py_InitModule("buffer_test", buffer_functions);
+#endif
+    if (m == NULL)
+        INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&PyMyArrayType) < 0)
+        INITERROR;
+    Py_INCREF(&PyMyArrayType);
+    PyModule_AddObject(m, "PyMyArray", (PyObject *)&PyMyArrayType);
+#if PY_MAJOR_VERSION >=3
+    return m;
+#endif
+}
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -9,7 +9,10 @@
             py.test.skip("unsupported before Python 2.7")
 
         w_hello = space.newbytes("hello")
+        assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
+        w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
+        assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
@@ -55,3 +58,15 @@
     @pytest.mark.skipif(True, reason='write a test for this')
     def test_get_base_and_get_buffer(self, space, api):
         assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER
+
+class AppTestBufferProtocol(AppTestCpythonExtensionBase):
+    def test_buffer_protocol(self):
+        import struct
+        module = self.import_module(name='buffer_test')
+        arr = module.PyMyArray(10)
+        y = memoryview(arr)
+        assert y.format == 'i'
+        assert y.shape == (10,)
+        s = y[3]
+        assert len(s) == struct.calcsize('i')
+        assert s == struct.pack('i', 3)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -17,7 +17,8 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, Py_buffer, PyTypeObject, PyTypeObjectPtr)
+    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -514,6 +515,7 @@
         bytes_getbuffer.api_func.get_wrapper(space))
     pto.c_tp_as_buffer = c_buf
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
+    pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def type_dealloc(space, obj):
@@ -681,6 +683,8 @@
             pto.c_tp_setattro = base.c_tp_setattro
         if not pto.c_tp_getattro:
             pto.c_tp_getattro = base.c_tp_getattro
+        if not pto.c_tp_as_buffer:
+            pto.c_tp_as_buffer = base.c_tp_as_buffer
     finally:
         Py_DecRef(space, base_pyo)
 
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -1,10 +1,11 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.lltypesystem.lltype import Ptr, FuncType, Void
 from pypy.module.cpyext.api import (cpython_struct, Py_ssize_t, Py_ssize_tP,
-    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_buffer,
+    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP,
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
+from pypy.module.cpyext.api import Py_bufferP
 
 
 P, FT, PyO = Ptr, FuncType, PyObject
@@ -53,7 +54,11 @@
 wrapperfunc = P(FT([PyO, PyO, rffi.VOIDP], PyO))
 wrapperfunc_kwds = P(FT([PyO, PyO, rffi.VOIDP, PyO], PyO))
 
-getbufferproc = P(FT([PyO, Ptr(Py_buffer), rffi.INT_real], rffi.INT_real))
+readbufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
+writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
+segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
+charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
+getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = P(FT([PyO, Ptr(Py_buffer)], Void))
 
 
@@ -126,6 +131,10 @@
 ))
 
 PyBufferProcs = cpython_struct("PyBufferProcs", (
+    ("bf_getreadbuffer", readbufferproc),
+    ("bf_getwritebuffer", writebufferproc),
+    ("bf_getsegcount", segcountproc),
+    ("bf_getcharbuffer", charbufferproc),
     ("bf_getbuffer", getbufferproc),
     ("bf_releasebuffer", releasebufferproc),
 ))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -19,7 +19,6 @@
     UserDelAction)
 from pypy.interpreter.pyframe import PyFrame
 
-
 class BogusBytecode(Exception):
     pass
 
@@ -383,6 +382,9 @@
         # XXX even the hacks have hacks
         if s == 'size': # used in _array() but never called by tests
             return IntObject(0)
+        if s == '__buffer__':
+            # descr___buffer__ does not exist on W_Root
+            return self.w_None
         return getattr(w_obj, 'descr_' + s)(self, *args)
 
     @specialize.arg(1)
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -704,3 +704,20 @@
     def get_raw_address(self):
         from rpython.rtyper.lltypesystem import rffi
         return rffi.ptradd(self.impl.storage, self.impl.start)
+
+    def getformat(self):
+        return self.impl.dtype.char
+
+    def getitemsize(self):
+        return self.impl.dtype.elsize
+
+    def getndim(self):
+        return len(self.impl.shape)
+
+    def getshape(self):
+        return self.impl.shape
+
+    def getstrides(self):
+        return self.impl.strides
+
+
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -468,7 +468,8 @@
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_buffer = space.getattr(w_buffer, space.wrap('__buffer__'))
+        w_buffer = space.call_method(w_buffer, '__buffer__', 
+                                    space.newint(space.BUF_FULL_RO))
         buf = _getbuffer(space, w_buffer)
 
     ts = buf.getlength()
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3617,13 +3617,35 @@
         assert str(exc.value) == "assignment destination is read-only"
 
         class A(object):
-            __buffer__ = 'abc'
+            def __buffer__(self, flags):
+                return 'abc'
 
         data = A()
         a = np.frombuffer(data, 'c')
         #assert a.base is data.__buffer__
         assert a.tostring() == 'abc'
 
+    def test_memoryview(self):
+        import numpy as np
+        import sys
+        if sys.version_info[:2] > (3, 2):
+            # In Python 3.3 the representation of empty shape, strides and sub-offsets
+            # is an empty tuple instead of None.
+            # http://docs.python.org/dev/whatsnew/3.3.html#api-changes
+            EMPTY = ()
+        else:
+            EMPTY = None
+        x = np.array([1, 2, 3, 4, 5], dtype='i')
+        y = memoryview('abc')
+        assert y.format == 'B'
+        y = memoryview(x)
+        assert y.format == 'i'
+        assert y.shape == (5,)
+        assert y.ndim == 1
+        assert y.strides == (4,)
+        assert y.suboffsets == EMPTY
+        assert y.itemsize == 4
+
     def test_fromstring(self):
         import sys
         from numpy import fromstring, dtype
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -423,7 +423,7 @@
         guard_false(i114, descr=...)
         --TICK--
         i123 = arraylen_gc(p67, descr=<ArrayP .>)
-        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=5 OS=110>)
+        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=<Calli . i EF=5 OS=110>)
         check_memory_error(i119)
         raw_store(i119, 0, i160, descr=<ArrayS 2>)
         raw_store(i119, 2, i160, descr=<ArrayS 2>)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1963,3 +1963,21 @@
                        ffi, "test_function_returns_opaque", "?")
     assert str(e.value) == ("function foo: 'struct a' is used as result type,"
                             " but is opaque")
+
+def test_function_returns_union():
+    ffi = FFI()
+    ffi.cdef("union u1 { int a, b; }; union u1 f1(int);")
+    lib = verify(ffi, "test_function_returns_union", """
+        union u1 { int a, b; };
+        static union u1 f1(int x) { union u1 u; u.b = x; return u; }
+    """)
+    assert lib.f1(51).a == 51
+
+def test_function_returns_partial_struct():
+    ffi = FFI()
+    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    lib = verify(ffi, "test_function_returns_partial_struct", """
+        struct a { int b, a, c; };
+        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+    """)
+    assert lib.f1(52).a == 52
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -423,6 +423,18 @@
         space.check_buf_flags(flags, True)
         return StringBuffer(self._value)
 
+    def readbuf_w(self, space):
+        return StringBuffer(self._value)
+
+    def writebuf_w(self, space):
+        raise oefmt(space.w_TypeError,
+                    "Cannot use string as modifiable buffer")
+
+    def descr_getbuffer(self, space, w_flags):
+        #from pypy.objspace.std.bufferobject import W_Buffer
+        #return W_Buffer(StringBuffer(self._value))
+        return self
+
     def listview_int(self):
         return _create_list_from_bytes(self._value)
 
@@ -654,11 +666,6 @@
     def descr_upper(self, space):
         return W_BytesObject(self._value.upper())
 
-    def descr_hex(self, space):
-        from pypy.objspace.std.bytearrayobject import _array_to_hexstring
-        return _array_to_hexstring(space, self.buffer_w(space, space.BUF_SIMPLE))
-
-
 
 def _create_list_from_bytes(value):
     # need this helper function to allow the jit to look inside and inline
@@ -838,7 +845,6 @@
 
     fromhex = interp2app(W_BytesObject.descr_fromhex, as_classmethod=True),
     maketrans = interp2app(W_BytesObject.descr_maketrans, as_classmethod=True),
-    hex = interp2app(W_BytesObject.descr_hex)
 )
 W_BytesObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1030,4 +1030,5 @@
     if x >= HASH_MODULUS:
         x -= HASH_MODULUS
 
-    return intmask(intmask(x) * sign)
+    h = intmask(intmask(x) * sign)
+    return h - (h == -1)
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -254,12 +254,12 @@
     def _make_generic_descr_binop(opname):
         if opname not in COMMUTATIVE_OPS:
             raise Exception("Not supported")
-            
+
         methname = opname + '_' if opname in ('and', 'or') else opname
         descr_rname = 'descr_r' + opname
         op = getattr(rbigint, methname)
         intop = getattr(rbigint, "int_" + methname)
-        
+
         @func_renamer('descr_' + opname)
         def descr_binop(self, space, w_other):
             if isinstance(w_other, W_IntObject):
@@ -279,7 +279,7 @@
             return W_LongObject(op(w_other.asbigint(), self.num))
 
         return descr_binop, descr_rbinop
-        
+
     descr_add, descr_radd = _make_generic_descr_binop('add')
     descr_sub, descr_rsub = _make_generic_descr_binop_noncommutative('sub')
     descr_mul, descr_rmul = _make_generic_descr_binop('mul')
@@ -321,12 +321,12 @@
         except OverflowError:   # b too big
             raise oefmt(space.w_OverflowError, "shift count too large")
         return W_LongObject(self.num.lshift(shift))
-        
+
     def _int_lshift(self, space, w_other):
         if w_other < 0:
             raise oefmt(space.w_ValueError, "negative shift count")
         return W_LongObject(self.num.lshift(w_other))
-        
+
     descr_lshift, descr_rlshift = _make_descr_binop(_lshift, _int_lshift)
 
     def _rshift(self, space, w_other):
@@ -337,7 +337,7 @@
         except OverflowError:   # b too big # XXX maybe just return 0L instead?
             raise oefmt(space.w_OverflowError, "shift count too large")
         return newlong(space, self.num.rshift(shift))
-        
+
     def _int_rshift(self, space, w_other):
         if w_other < 0:
             raise oefmt(space.w_ValueError, "negative shift count")
@@ -352,7 +352,7 @@
             raise oefmt(space.w_ZeroDivisionError,
                         "long division or modulo by zero")
         return newlong(space, z)
-        
+
     def _floordiv(self, space, w_other):
         try:
             z = self.num.floordiv(w_other.asbigint())
@@ -369,7 +369,7 @@
             raise oefmt(space.w_ZeroDivisionError,
                         "integer division or modulo by zero")
         return newlong(space, z)
-        
+
     def _int_mod(self, space, w_other):
         try:
             z = self.num.int_mod(w_other)
@@ -404,7 +404,8 @@
         while x >= HASH_MODULUS:
             x -= HASH_MODULUS
         i -= 1
-    return intmask(intmask(x) * v.sign)
+    h = intmask(intmask(x) * v.sign)
+    return h - (h == -1)
 
 
 def newlong(space, bigint):
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -82,18 +82,29 @@
     def descr_getitem(self, space, w_index):
         self._check_released(space)
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        itemsize = self.buf.getitemsize()
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+            stop  = start + size
+            if step == 0:
+                step = 1
+            if stop > self.getlength():
+                raise oefmt(space.w_IndexError, 'index out of range')
+        if step not in (0, 1):
+            raise oefmt(space.w_NotImplementedError, "")
         if step == 0:  # index only
             # TODO: this probably isn't very fast
-            buf = SubBuffer(self.buf, start * self.itemsize, self.itemsize)
+            buf = SubBuffer(self.buf, start, self.itemsize)
             fmtiter = UnpackFormatIterator(space, buf)
             fmtiter.interpret(self.format)
             return fmtiter.result_w[0]
         elif step == 1:
-            buf = SubBuffer(self.buf, start * self.itemsize,
-                            size * self.itemsize)
+            buf = SubBuffer(self.buf, start, size)
             return W_MemoryView(buf, self.format, self.itemsize)
         else:
-            raise oefmt(space.w_NotImplementedError, "")
+            buf = SubBuffer(self.buf, start, size)
+            return W_MemoryView(buf)
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
@@ -102,6 +113,21 @@
         if space.isinstance_w(w_index, space.w_tuple):
             raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        itemsize = self.buf.getitemsize()
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+            stop  = start + size
+            if step == 0:
+                step = 1
+            if stop > self.getlength():
+                raise oefmt(space.w_IndexError, 'index out of range')
+        if step not in (0, 1):
+            raise oefmt(space.w_NotImplementedError, "")
+        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+        if value.getlength() != size:
+            raise oefmt(space.w_ValueError,
+                        "cannot modify size of memoryview object")
         if step == 0:  # index only
             # TODO: this probably isn't very fast
             fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)
@@ -111,43 +137,43 @@
                 raise oefmt(space.w_TypeError,
                             "memoryview: invalid type for format '%s'",
                             self.format)
-            self.buf.setslice(start * self.itemsize, fmtiter.result.build())
+            self.buf.setslice(start, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
             if value.getlength() != size * self.itemsize:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
-            self.buf.setslice(start * self.itemsize, value.as_str())
+            self.buf.setslice(start, value.as_str())
         else:
             raise oefmt(space.w_NotImplementedError, "")
 
     def descr_len(self, space):
         self._check_released(space)
-        return space.wrap(self.getlength())
+        return space.wrap(self.buf.getlength())
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap(self.format)
+        return space.wrap(self.buf.getformat())
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.wrap(self.itemsize)
+        return space.wrap(self.buf.getitemsize())
 
     def w_get_ndim(self, space):
         self._check_released(space)
-        return space.wrap(1)
+        return space.wrap(self.buf.getndim())
 
     def w_is_readonly(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.readonly)
+        return space.newbool(bool(self.buf.readonly))
 
     def w_get_shape(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(self.getlength())])
+        return space.newtuple([space.wrap(x) for x in self.buf.getshape()])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(self.itemsize)])
+        return space.newtuple([space.wrap(x) for x in self.buf.getstrides()])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)
@@ -237,7 +263,6 @@
         self._check_released(space)
         return _array_to_hexstring(space, self.buf)
 
-
 W_MemoryView.typedef = TypeDef(
     "memoryview",
     __doc__ = """\
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -265,7 +265,11 @@
         result = f1.int(self.space)
         assert result == f1
 
-class AppTestInt:
+class AppTestInt(object):
+    def test_hash(self):
+        assert hash(-1) == (-1).__hash__() == -2
+        assert hash(-2) == (-2).__hash__() == -2
+
     def test_conjugate(self):
         assert (1).conjugate() == 1
         assert (-1).conjugate() == -1
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -122,6 +122,9 @@
     def newlist(self, iterable):
         return list(iterable)
 
+    def newbytes(self, obj):
+        return bytes(obj)
+
     def call_function(self, func, *args, **kwds):
         return func(*args, **kwds)
 
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -261,6 +261,9 @@
                    "stack based virtual machines (only for backends that support it)",
                    default=True),
         BoolOption("storesink", "Perform store sinking", default=True),
+        BoolOption("replace_we_are_jitted",
+                   "Replace we_are_jitted() calls by False",
+                   default=False, cmdline=None),
         BoolOption("none",
                    "Do not run any backend optimizations",
                    requires=[('translation.backendopt.inline', False),
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -833,9 +833,6 @@
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     # vector operations
     vector_arith_code = """
     def bh_vec_{0}_{1}(self, vx, vy, count):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -757,9 +757,6 @@
             self.write_int_at_mem(res, self.vtable_offset, WORD, sizedescr.get_vtable())
         return res
 
-    def bh_new_raw_buffer(self, size):
-        return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
-
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -229,8 +229,6 @@
         raise NotImplementedError
     def bh_newunicode(self, length):
         raise NotImplementedError
-    def bh_new_raw_buffer(self, size):
-        raise NotImplementedError
 
     def bh_arraylen_gc(self, array, arraydescr):
         raise NotImplementedError
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -582,6 +582,14 @@
                 return lltype.malloc(ARRAY, n, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_1_raw_malloc_varsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_1_raw_malloc_varsize.func_name = name
             return _ll_1_raw_malloc_varsize
         return build_ll_1_raw_malloc_varsize
 
@@ -610,6 +618,14 @@
                 return lltype.malloc(STRUCT, flavor='raw', zero=zero,
                                      add_memory_pressure=add_memory_pressure,
                                      track_allocation=track_allocation)
+            name = '_ll_0_raw_malloc_fixedsize'
+            if zero:
+                name += '_zero'
+            if add_memory_pressure:
+                name += '_mpressure'
+            if not track_allocation:
+                name += '_notrack'
+            _ll_0_raw_malloc_fixedsize.func_name = name
             return _ll_0_raw_malloc_fixedsize
         return build_ll_0_raw_malloc_fixedsize
 
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -367,8 +367,9 @@
 
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
-    
-    def __init__(self, cpu, size=-1):
+
+    def __init__(self, cpu, func, size=-1):
+        self.func = func
         self.size = size
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
@@ -425,7 +426,8 @@
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
         buffer = self._get_buffer()
-        return visitor.visit_vrawbuffer(self.size,
+        return visitor.visit_vrawbuffer(self.func,
+                                        self.size,
                                         buffer.offsets[:],
                                         buffer.descrs[:])
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1770,7 +1770,7 @@
     def test_virtual_raw_malloc_basic(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)   # 12345 = malloc func
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1787,7 +1787,7 @@
         ops = """
         [i1]
         i5 = int_mul(10, 1)
-        i2 = call_i('malloc', i5, descr=raw_malloc_descr)
+        i2 = call_i(12345, i5, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
@@ -1803,7 +1803,7 @@
     def test_virtual_raw_malloc_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1817,7 +1817,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i(12345, 20, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr_char)
         raw_store(i2, 1, 123, descr=rawarraydescr_char)
@@ -1832,7 +1832,7 @@
     def test_virtual_raw_malloc_invalid_write_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1843,7 +1843,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1855,7 +1855,7 @@
     def test_virtual_raw_malloc_invalid_read_force(self):
         ops = """
         [i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label(i1) # we expect the buffer to be forced *after* the label
@@ -1866,7 +1866,7 @@
         expected = """
         [i1]
         label(i1)
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, i1, descr=rawarraydescr)
         i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr_char)
@@ -1878,7 +1878,7 @@
     def test_virtual_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1898,7 +1898,7 @@
     def test_virtual_raw_slice_of_a_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         i3 = int_add(i2, 1) # get a slice of the original buffer
         i4 = int_add(i3, 1) # get a slice of a slice
@@ -1916,7 +1916,7 @@
     def test_virtual_raw_slice_force(self):
         ops = """
         [i0, i1]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1929,7 +1929,7 @@
         [i0, i1]
         label(i0, i1)
         # these ops are generated by VirtualRawBufferValue._really_force
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         raw_store(i2, 0, 42, descr=rawarraydescr_char)
         raw_store(i2, 5, 4242, descr=rawarraydescr_char)
@@ -1946,7 +1946,7 @@
         i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr)
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i3, 0, i2, descr=rawarraydescr)
         label(i2)
@@ -1958,7 +1958,7 @@
         i2 = int_add(i1, 1)
         call_n('free', i0, descr=raw_free_descr)
         label(i2)
-        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i3)
         raw_store(i3, 0, i2, descr=rawarraydescr)
         jump(i3)
@@ -1968,7 +1968,7 @@
     def test_virtual_raw_store_raw_load(self):
         ops = """
         [i1]
-        i0 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i0 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 0, i1, descr=rawarraydescr)
         i2 = raw_load_i(i0, 0, descr=rawarraydescr)
@@ -1986,7 +1986,7 @@
     def test_virtual_raw_store_getarrayitem_raw(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 8, f1, descr=rawarraydescr_float)
         f2 = getarrayitem_raw_f(i0, 1, descr=rawarraydescr_float)
@@ -2004,7 +2004,7 @@
     def test_virtual_setarrayitem_raw_raw_load(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i0, 1, f1, descr=rawarraydescr_float)
         f2 = raw_load_f(i0, 8, descr=rawarraydescr_float)
@@ -2022,7 +2022,7 @@
     def test_virtual_raw_buffer_forced_but_slice_not_forced(self):
         ops = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         guard_no_exception() []
         i1 = int_add(i0, 8)
         escape_n(i0)
@@ -2031,7 +2031,7 @@
         """
         expected = """
         [f1]
-        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i(12345, 16, descr=raw_malloc_descr)
         check_memory_error(i0)
         escape_n(i0)
         i1 = int_add(i0, 8)
@@ -8886,7 +8886,7 @@
     def test_resume_forced_raw_ptr(self):
         ops = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         is = int_add(i, 8)
         escape_n(i)
@@ -8898,7 +8898,7 @@
         """
         expected = """
         [i0]
-        i = call_i('malloc', 10, descr=raw_malloc_descr)
+        i = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i)
         escape_n(i)
         i1 = int_add(i0, 1)
@@ -8966,7 +8966,7 @@
     def test_pending_setfield_delayed_malloc(self):
         ops = """
         [i0, p0]
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 13, descr=rawarraydescr)
         setfield_gc(p0, i2, descr=valuedescr)
@@ -8988,14 +8988,14 @@
     def test_raw_buffer_ptr_info_intbounds_bug(self):
         ops = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         guard_no_exception() []
         guard_value(i2, 12345) []
         jump()
         """
         expected = """
         []
-        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i(12345, 10, descr=raw_malloc_descr)
         check_memory_error(i2)
         guard_value(i2, 12345) []
         jump()
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -45,7 +45,8 @@
         return opinfo
 
     def make_virtual_raw_memory(self, size, source_op):
-        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, size)
+        func = source_op.getarg(0).getint()
+        opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, func, size)
         newop = self.replace_op_with(source_op, source_op.getopnum(),
                                      args=[source_op.getarg(0), ConstInt(size)])
         newop.set_forwarded(opinfo)
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -364,8 +364,8 @@
     def visit_varraystruct(self, arraydescr, size, fielddescrs):
         return VArrayStructInfo(arraydescr, size, fielddescrs)
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
-        return VRawBufferInfo(size, offsets, descrs)
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
+        return VRawBufferInfo(func, size, offsets, descrs)
 
     def visit_vrawslice(self, offset):
         return VRawSliceInfo(offset)
@@ -703,7 +703,8 @@
 
 class VRawBufferInfo(VAbstractRawInfo):
 
-    def __init__(self, size, offsets, descrs):
+    def __init__(self, func, size, offsets, descrs):
+        self.func = func
         self.size = size
         self.offsets = offsets
         self.descrs = descrs
@@ -711,7 +712,7 @@
     @specialize.argtype(1)
     def allocate_int(self, decoder, index):
         length = len(self.fieldnums)
-        buffer = decoder.allocate_raw_buffer(self.size)
+        buffer = decoder.allocate_raw_buffer(self.func, self.size)
         decoder.virtuals_cache.set_int(index, buffer)
         for i in range(len(self.offsets)):
             offset = self.offsets[i]
@@ -1130,9 +1131,13 @@
                                                            lengthbox)
         return self.metainterp.execute_new_array(arraydescr, lengthbox)
 
-    def allocate_raw_buffer(self, size):
+    def allocate_raw_buffer(self, func, size):
         cic = self.metainterp.staticdata.callinfocollection
-        calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        # Can't use 'func' from callinfo_for_oopspec(), because we have
+        # several variants (zero/non-zero, memory-pressure or not, etc.)
+        # and we have to pick the correct one here; that's why we save
+        # it in the VRawBufferInfo.
         return self.metainterp.execute_and_record_varargs(
             rop.CALL_I, [ConstInt(func), ConstInt(size)], calldescr)
 
@@ -1461,10 +1466,11 @@
     def allocate_string(self, length):
         return self.cpu.bh_newstr(length)
 
-    def allocate_raw_buffer(self, size):
-        buffer = self.cpu.bh_new_raw_buffer(size)
-        adr = llmemory.cast_ptr_to_adr(buffer)
-        return llmemory.cast_adr_to_int(adr, "symbolic")
+    def allocate_raw_buffer(self, func, size):
+        from rpython.jit.codewriter import heaptracker
+        cic = self.callinfocollection
+        calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)
+        return self.cpu.bh_call_i(func, [size], None, None, calldescr)
 
     def string_setitem(self, str, index, charnum):
         char = self.decode_int(charnum)
diff --git a/rpython/jit/metainterp/walkvirtual.py b/rpython/jit/metainterp/walkvirtual.py
--- a/rpython/jit/metainterp/walkvirtual.py
+++ b/rpython/jit/metainterp/walkvirtual.py
@@ -17,7 +17,7 @@
     def visit_varraystruct(self, arraydescr, fielddescrs):
         raise NotImplementedError("abstract base class")
 
-    def visit_vrawbuffer(self, size, offsets, descrs):
+    def visit_vrawbuffer(self, func, size, offsets, descrs):
         raise NotImplementedError("abstract base class")
 
     def visit_vrawslice(self, offset):
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -452,7 +452,8 @@
                               merge_if_blocks=True,
                               constfold=True,
                               remove_asserts=True,
-                              really_remove_asserts=True)
+                              really_remove_asserts=True,
+                              replace_we_are_jitted=False)
 
     def prejit_optimizations_minimal_inline(self, policy, graphs):
         from rpython.translator.backendopt.inline import auto_inline_graphs
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -59,6 +59,20 @@
     def get_raw_address(self):
         raise ValueError("no raw buffer")
 
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
 
 class StringBuffer(Buffer):
     __slots__ = ['value']
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -799,8 +799,8 @@
                            rffi.cast(size_t, map_size),
                            rffi.cast(rffi.INT, use_flag))
     else:
-        def madvice_free(addr, map_size):
-            "No madvice() on this platform"
+        def madvise_free(addr, map_size):
+            "No madvise() on this platform"
 
 elif _MS_WINDOWS:
     def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -90,6 +90,9 @@
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
+        attrs = CodeClass.__dict__.get('_attrs_', None)
+        if attrs is not None:
+            CodeClass._attrs_ = list(attrs) + ['_vmprof_unique_id']
         self._code_classes.add(CodeClass)
         #
         class WeakCodeObjectList(RWeakListMixin):
@@ -189,7 +192,7 @@
 
         def decorated_function(*args):
             unique_id = get_code_fn(*args)._vmprof_unique_id
-            unique_id = rffi.cast(lltype.Signed, unique_id) 
+            unique_id = rffi.cast(lltype.Signed, unique_id)
             # ^^^ removes the "known non-negative" hint for annotation
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)
diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py
--- a/rpython/translator/backendopt/all.py
+++ b/rpython/translator/backendopt/all.py
@@ -2,6 +2,7 @@
 from rpython.translator.backendopt import inline
 from rpython.translator.backendopt.malloc import remove_mallocs
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.translator.backendopt.stat import print_statistics
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator import simplify
@@ -36,6 +37,7 @@
     # inline_threshold, mallocs
     # merge_if_blocks, constfold, heap2stack
     # clever_malloc_removal, remove_asserts
+    # replace_we_are_jitted
 
     config = translator.config.translation.backendopt.copy(as_default=True)
     config.set(**kwds)
@@ -49,6 +51,10 @@
         print "before optimizations:"
         print_statistics(translator.graphs[0], translator, "per-graph.txt")
 
+    if config.replace_we_are_jitted:
+        for graph in graphs:
+            replace_we_are_jitted(graph)
+
     if config.remove_asserts:
         constfold(config, graphs)
         remove_asserts(translator, graphs)
diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py
--- a/rpython/translator/backendopt/constfold.py
+++ b/rpython/translator/backendopt/constfold.py
@@ -276,3 +276,25 @@
             rewire_links(splitblocks, graph)
         if not diffused and not splitblocks:
             break # finished
+
+def replace_symbolic(graph, symbolic, value):
+    result = False
+    for block in graph.iterblocks():
+        for op in block.operations:
+            for i, arg in enumerate(op.args):
+                if isinstance(arg, Constant) and arg.value is symbolic:
+                    op.args[i] = value
+                    result = True
+        if block.exitswitch is symbolic:
+            block.exitswitch = value
+            result = True
+    return result
+
+def replace_we_are_jitted(graph):
+    from rpython.rlib import jit
+    replacement = Constant(0)
+    replacement.concretetype = lltype.Signed
+    did_replacement = replace_symbolic(graph, jit._we_are_jitted, replacement)
+    if did_replacement:
+        constant_fold_graph(graph)
+    return did_replacement
diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py
--- a/rpython/translator/backendopt/test/test_all.py
+++ b/rpython/translator/backendopt/test/test_all.py
@@ -289,3 +289,19 @@
         llinterp = LLInterpreter(t.rtyper)
         res = llinterp.eval_graph(later_graph, [10])
         assert res == 1
+
+    def test_replace_we_are_jitted(self):
+        from rpython.rlib import jit
+        def f():
+            if jit.we_are_jitted():
+                return 1
+            return 2 + jit.we_are_jitted()
+
+        t = self.translateopt(f, [])
+        graph = graphof(t, f)
+        # by default, replace_we_are_jitted is off
+        assert graph.startblock.operations[0].args[0].value is jit._we_are_jitted
+
+        t = self.translateopt(f, [], replace_we_are_jitted=True)
+        graph = graphof(t, f)
+        assert graph.startblock.exits[0].args[0].value == 2
diff --git a/rpython/translator/backendopt/test/test_constfold.py b/rpython/translator/backendopt/test/test_constfold.py
--- a/rpython/translator/backendopt/test/test_constfold.py
+++ b/rpython/translator/backendopt/test/test_constfold.py
@@ -7,6 +7,7 @@
 from rpython.rtyper import rclass
 from rpython.rlib import objectmodel
 from rpython.translator.backendopt.constfold import constant_fold_graph
+from rpython.translator.backendopt.constfold import replace_we_are_jitted
 from rpython.conftest import option
 
 def get_graph(fn, signature):
@@ -343,3 +344,18 @@
     merge_if_blocks.merge_if_blocks_once(graph)
     constant_fold_graph(graph)
     check_graph(graph, [], 66, t)
+
+def test_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def fn():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+    graph, t = get_graph(fn, [])
+    result = replace_we_are_jitted(graph)
+    assert result
+    checkgraph(graph)
+    # check shape of graph
+    assert len(graph.startblock.operations) == 0
+    assert graph.startblock.exitswitch is None
+    assert graph.startblock.exits[0].target.exits[0].args[0].value == 2
diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -381,7 +381,7 @@
         """ Run all backend optimizations - lltype version
         """
         from rpython.translator.backendopt.all import backend_optimizations
-        backend_optimizations(self.translator)
+        backend_optimizations(self.translator, replace_we_are_jitted=True)
 
 
     STACKCHECKINSERTION = 'stackcheckinsertion_lltype'
diff --git a/rpython/translator/test/test_interactive.py b/rpython/translator/test/test_interactive.py
--- a/rpython/translator/test/test_interactive.py
+++ b/rpython/translator/test/test_interactive.py
@@ -78,3 +78,15 @@
     dll = ctypes.CDLL(str(t.driver.c_entryp))
     f = dll.pypy_g_f
     assert f(2, 3) == 5
+
+def test_check_that_driver_uses_replace_we_are_jitted():
+    from rpython.rlib import jit
+    def f():
+        if jit.we_are_jitted():
+            return 1
+        return 2 + jit.we_are_jitted()
+
+    t = Translation(f, [])
+    t.backendopt()
+    graph = t.driver.translator.graphs[0]
+    assert graph.startblock.exits[0].args[0].value == 2

From pypy.commits at gmail.com  Wed Aug 24 05:00:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 02:00:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: resolve some merge conflicts
Message-ID: <57bd623f.a111c20a.54f43.7827@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86461:7011e843b4a7
Date: 2016-08-24 10:34 +0200
http://bitbucket.org/pypy/pypy/changeset/7011e843b4a7/

Log:	resolve some merge conflicts

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -208,7 +208,7 @@
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
-    address = address & (sys.maxint * 2 + 1)
+    address = address & (sys.maxsize * 2 + 1)
     instance = self.__new__(self)
     lgt = getattr(self, '_length_', 1)
     instance._buffer = self._ffiarray.fromaddress(address, lgt)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -585,11 +585,6 @@
         self.sys = Module(self, w_name)
         self.sys.install()
 
-        from pypy.module.imp import Module
-        w_name = self.wrap('imp')
-        mod = Module(self, w_name)
-        mod.install()
-
         from pypy.module.__builtin__ import Module
         w_name = self.wrap('builtins')
         self.builtin = Module(self, w_name)
@@ -1998,7 +1993,7 @@
 
 ObjSpace.IrregularOpTable = [
     'wrap',
-    'str_w',
+    'bytes_w',
     'int_w',
     'float_w',
     'uint_w',
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.lltypesystem.lltype import Ptr, FuncType, Void
 from pypy.module.cpyext.api import (cpython_struct, Py_ssize_t, Py_ssize_tP,
-    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP,
+    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_Buffer,
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
@@ -59,7 +59,7 @@
 segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
 charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
 getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
-releasebufferproc = P(FT([PyO, Ptr(Py_buffer)], Void))
+releasebufferproc = P(FT([PyO, Py_bufferP], Void))
 
 
 PyGetSetDef = cpython_struct("PyGetSetDef", (

From pypy.commits at gmail.com  Wed Aug 24 05:00:48 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 02:00:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: pull some merge conflicts from py3k
Message-ID: <57bd6240.d8011c0a.577cb.b88d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86462:74d03f9c65e8
Date: 2016-08-24 10:34 +0200
http://bitbucket.org/pypy/pypy/changeset/74d03f9c65e8/

Log:	pull some merge conflicts from py3k

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -208,7 +208,7 @@
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
-    address = address & (sys.maxint * 2 + 1)
+    address = address & (sys.maxsize * 2 + 1)
     instance = self.__new__(self)
     lgt = getattr(self, '_length_', 1)
     instance._buffer = self._ffiarray.fromaddress(address, lgt)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -585,11 +585,6 @@
         self.sys = Module(self, w_name)
         self.sys.install()
 
-        from pypy.module.imp import Module
-        w_name = self.wrap('imp')
-        mod = Module(self, w_name)
-        mod.install()
-
         from pypy.module.__builtin__ import Module
         w_name = self.wrap('builtins')
         self.builtin = Module(self, w_name)
@@ -2001,7 +1996,7 @@
 
 ObjSpace.IrregularOpTable = [
     'wrap',
-    'str_w',
+    'bytes_w',
     'int_w',
     'float_w',
     'uint_w',
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.lltypesystem.lltype import Ptr, FuncType, Void
 from pypy.module.cpyext.api import (cpython_struct, Py_ssize_t, Py_ssize_tP,
-    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP,
+    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_Buffer,
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
@@ -59,7 +59,7 @@
 segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
 charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
 getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
-releasebufferproc = P(FT([PyO, Ptr(Py_buffer)], Void))
+releasebufferproc = P(FT([PyO, Py_bufferP], Void))
 
 
 PyGetSetDef = cpython_struct("PyGetSetDef", (

From pypy.commits at gmail.com  Wed Aug 24 05:00:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 02:00:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: cpyext, struct to usemodules
Message-ID: <57bd6243.04141c0a.e63b0.ac2f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86463:f8a9272bbf66
Date: 2016-08-24 11:00 +0200
http://bitbucket.org/pypy/pypy/changeset/f8a9272bbf66/

Log:	cpyext, struct to usemodules

diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -79,7 +79,7 @@
 
 
 class AppTestRecompiler:
-    spaceconfig = dict(usemodules=['_cffi_backend', 'imp'])
+    spaceconfig = dict(usemodules=['_cffi_backend', 'imp', 'cpyext', 'struct'])
 
     def setup_class(cls):
         if cls.runappdirect:

From pypy.commits at gmail.com  Wed Aug 24 05:37:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 02:37:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Update the comments about the reason for
 duplicating these few lines of
Message-ID: <57bd6ac8.d42f1c0a.d0cb9.beb5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86465:7f8f51b42ad0
Date: 2016-08-24 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/7f8f51b42ad0/

Log:	Update the comments about the reason for duplicating these few lines
	of code from lib-python/3/code.py.

diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -49,9 +49,11 @@
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
-    # some parts of code.py are copied here because it seems to be impossible
+    # some parts of code.py are copied here because it was impossible
     # to start an interactive console without printing at least one line
-    # of banner
+    # of banner.  This was fixed in 3.4; but then from 3.6 it prints a
+    # line when exiting.  This can be disabled too---by passing an argument
+    # that doesn't exist in <= 3.5.  So, too much mess: just copy the code.
     more = 0
     while 1:
         try:

From pypy.commits at gmail.com  Wed Aug 24 05:41:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 02:41:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merged in marky1991/pypy_new/py3k (pull
 request #474)
Message-ID: <57bd6bc7.56421c0a.e1297.bb3a@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3k
Changeset: r86469:213acc3c585e
Date: 2016-08-24 11:40 +0200
http://bitbucket.org/pypy/pypy/changeset/213acc3c585e/

Log:	Merged in marky1991/pypy_new/py3k (pull request #474)

	Define clock_t on windows (py3k)

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -223,7 +223,7 @@
             pass
 
 if _WIN32:
-    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h']
+    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h']
     libraries = []
 else:
     if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')):
@@ -254,10 +254,10 @@
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
+    CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
-        CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),

From pypy.commits at gmail.com  Wed Aug 24 05:41:47 2016
From: pypy.commits at gmail.com (marky1991)
Date: Wed, 24 Aug 2016 02:41:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Define clock_t on windows too.
Message-ID: <57bd6bdb.8f8e1c0a.2f009.c6b7@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k
Changeset: r86466:9f32d69d4211
Date: 2016-08-23 17:38 -0400
http://bitbucket.org/pypy/pypy/changeset/9f32d69d4211/

Log:	Define clock_t on windows too.

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -220,7 +220,8 @@
             pass
 
 if _WIN32:
-    includes = ['io.h', 'sys/utime.h', 'sys/types.h']
+    includes = ['io.h', 'sys/utime.h', 'sys/types.h',
+                'time.h']
     libraries = []
 else:
     if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')):
@@ -250,9 +251,8 @@
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
+    CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
     if not _WIN32:
-        CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
-
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),
                            ('tms_stime', rffi.INT),

From pypy.commits at gmail.com  Wed Aug 24 05:41:49 2016
From: pypy.commits at gmail.com (marky1991)
Date: Wed, 24 Aug 2016 02:41:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merging from windows with linux side
Message-ID: <57bd6bdd.0cce1c0a.dc8d9.b1d1@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k
Changeset: r86467:c78024f7ecad
Date: 2016-08-23 17:40 -0400
http://bitbucket.org/pypy/pypy/changeset/c78024f7ecad/

Log:	Merging from windows with linux side

diff too long, truncating to 2000 out of 5058 lines

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -452,7 +452,7 @@
     RegrTest('test_userstring.py', core=True),
     RegrTest('test_uu.py'),
     RegrTest('test_uuid.py'),
-    RegrTest('test_venv.py'),
+    RegrTest('test_venv.py', usemodules="struct"),
     RegrTest('test_wait3.py', usemodules="thread"),
     RegrTest('test_wait4.py', usemodules="thread"),
     RegrTest('test_warnings.py', core=True),
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -15,8 +15,9 @@
 
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
-        includes=['unistd.h', 'sys/syscall.h'])
+        includes=['unistd.h', 'sys/syscall.h', 'sys/stat.h'])
     HAVE_SYS_SYSCALL_H = platform.Has("syscall")
+    HAVE_SYS_STAT_H = platform.Has("stat")
     HAVE_SETSID = platform.Has("setsid")
 
 config = platform.configure(CConfig)
@@ -29,6 +30,8 @@
 compile_extra = []
 if config['HAVE_SYS_SYSCALL_H']:
     compile_extra.append("-DHAVE_SYS_SYSCALL_H")
+if config['HAVE_SYS_STAT_H']:
+    compile_extra.append("-DHAVE_SYS_STAT_H")
 if config['HAVE_SETSID']:
     compile_extra.append("-DHAVE_SETSID")
 
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -159,7 +159,6 @@
         libraries=rtime.libraries
     )
     CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
-    clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
@@ -233,7 +232,6 @@
 HAS_CLOCK_MONOTONIC = cConfig.CLOCK_MONOTONIC is not None
 HAS_MONOTONIC = (_WIN or _MACOSX or
                  (HAS_CLOCK_GETTIME and (HAS_CLOCK_HIGHRES or HAS_CLOCK_MONOTONIC)))
-clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
 
@@ -1024,7 +1022,10 @@
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
                 if rffi.cast(lltype.Signed, ret) != -1:
-                    cpu_time = float(tms.c_tms_utime + tms.c_tms_stime)
+                    cpu_time = float(rffi.cast(lltype.Signed,
+                                               tms.c_tms_utime) +
+                                     rffi.cast(lltype.Signed,
+                                               tms.c_tms_stime))
                     if w_info is not None:
                         _setinfo(space, w_info, "times()",
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
@@ -1032,7 +1033,7 @@
                     return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
         return clock(space)
 
-_clock = external('clock', [], clock_t)
+_clock = external('clock', [], rposix.CLOCK_T)
 def clock(space, w_info=None):
     """clock() -> floating point number
 
@@ -1046,7 +1047,7 @@
             pass
     value = _clock()
     # Is this casting correct?
-    if value == rffi.cast(clock_t, -1):
+    if intmask(value) == intmask(rffi.cast(rposix.CLOCK_T, -1)):
         raise oefmt(space.w_RuntimeError,
                     "the processor time used is not available or its value"
                     "cannot be represented")
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -211,11 +211,10 @@
 
     def test_identity_vs_id_primitives(self):
         import sys
-        l = range(-10, 10, 2)
+        l = list(range(-10, 10, 2))
         for i in [0, 1, 3]:
             l.append(float(i))
             l.append(i + 0.1)
-            l.append(long(i))
             l.append(i + sys.maxsize)
             l.append(i - sys.maxsize)
             l.append(i + 1j)
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -976,3 +976,12 @@
         raises(TypeError, "u''.encode(None)")
         raises(TypeError, "str(b'', encoding=None)")
         raises(TypeError, 'u"".encode("utf-8", None)')
+
+    def test_casefold(self):
+        assert 'hello'.casefold() == 'hello'
+        assert 'hELlo'.casefold() == 'hello'
+        assert '&#223;'.casefold() == 'ss'
+        assert '&#64257;'.casefold() == 'fi'
+        assert '\u03a3'.casefold() == '\u03c3'
+        assert 'A\u0345\u03a3'.casefold() == 'a\u03b9\u03c3'
+        assert '\u00b5'.casefold() == '\u03bc'
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -414,6 +414,19 @@
     def _join_check_item(self, space, w_obj):
         return not space.isinstance_w(w_obj, space.w_unicode)
 
+    def descr_casefold(self, space):
+        value = self._val(space)
+        builder = self._builder(len(value))
+        for c in value:
+            c_ord = ord(c)
+            folded = unicodedb.casefold_lookup(c_ord)
+            if folded is None:
+                builder.append(unichr(unicodedb.tolower(c_ord)))
+            else:
+                for r in folded:
+                    builder.append(unichr(r))
+        return self._new(builder.build())
+
     def descr_isdecimal(self, space):
         return self._is_generic(space, '_isdecimal')
 
@@ -815,6 +828,12 @@
         and there is at least one character in S, False otherwise.
         """
 
+    def casefold():
+        """S.casefold() -> str
+
+        Return a version of S suitable for caseless comparisons.
+        """
+
     def isdecimal():
         """S.isdecimal() -> bool
 
@@ -1105,6 +1124,8 @@
 
     capitalize = interp2app(W_UnicodeObject.descr_capitalize,
                             doc=UnicodeDocstrings.capitalize.__doc__),
+    casefold = interp2app(W_UnicodeObject.descr_casefold,
+                            doc=UnicodeDocstrings.casefold.__doc__),
     center = interp2app(W_UnicodeObject.descr_center,
                         doc=UnicodeDocstrings.center.__doc__),
     count = interp2app(W_UnicodeObject.descr_count,
diff --git a/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt b/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt
@@ -0,0 +1,912 @@
+# CaseFolding-3.2.0.txt
+# Date: 2002-03-22,20:54:33 GMT [MD]
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#65533;e" to match.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, see
+# UTR #21 Case Mappings, at http://www.unicode.org/unicode/reports/tr21/
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F2; C; 03C3; # GREEK LUNATE SIGMA SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N
+24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O
+24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P
+24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q
+24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R
+24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S
+24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T
+24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U
+24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V
+24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W
+24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X
+24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y
+24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z
+FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF
+FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI
+FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
+FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI
+FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL
+FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T
+FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST
+FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW
+FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH
+FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI
+FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW
+FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH
+FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A
+FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B
+FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C
+FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D
+FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E
+FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F
+FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G
+FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H
+FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I
+FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J
+FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K
+FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L
+FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M
+FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N
+FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O
+FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P
+FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q
+FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R
+FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S
+FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T
+FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U
+FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V
+FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W
+FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X
+FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y
+FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
+10400; C; 10428; # DESERET CAPITAL LETTER LONG I
+10401; C; 10429; # DESERET CAPITAL LETTER LONG E
+10402; C; 1042A; # DESERET CAPITAL LETTER LONG A
+10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH
+10404; C; 1042C; # DESERET CAPITAL LETTER LONG O
+10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO
+10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I
+10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E
+10408; C; 10430; # DESERET CAPITAL LETTER SHORT A
+10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH
+1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O
+1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO
+1040C; C; 10434; # DESERET CAPITAL LETTER AY
+1040D; C; 10435; # DESERET CAPITAL LETTER OW
+1040E; C; 10436; # DESERET CAPITAL LETTER WU
+1040F; C; 10437; # DESERET CAPITAL LETTER YEE
+10410; C; 10438; # DESERET CAPITAL LETTER H
+10411; C; 10439; # DESERET CAPITAL LETTER PEE
+10412; C; 1043A; # DESERET CAPITAL LETTER BEE
+10413; C; 1043B; # DESERET CAPITAL LETTER TEE
+10414; C; 1043C; # DESERET CAPITAL LETTER DEE
+10415; C; 1043D; # DESERET CAPITAL LETTER CHEE
+10416; C; 1043E; # DESERET CAPITAL LETTER JEE
+10417; C; 1043F; # DESERET CAPITAL LETTER KAY
+10418; C; 10440; # DESERET CAPITAL LETTER GAY
+10419; C; 10441; # DESERET CAPITAL LETTER EF
+1041A; C; 10442; # DESERET CAPITAL LETTER VEE
+1041B; C; 10443; # DESERET CAPITAL LETTER ETH
+1041C; C; 10444; # DESERET CAPITAL LETTER THEE
+1041D; C; 10445; # DESERET CAPITAL LETTER ES
+1041E; C; 10446; # DESERET CAPITAL LETTER ZEE
+1041F; C; 10447; # DESERET CAPITAL LETTER ESH
+10420; C; 10448; # DESERET CAPITAL LETTER ZHEE
+10421; C; 10449; # DESERET CAPITAL LETTER ER
+10422; C; 1044A; # DESERET CAPITAL LETTER EL
+10423; C; 1044B; # DESERET CAPITAL LETTER EM
+10424; C; 1044C; # DESERET CAPITAL LETTER EN
+10425; C; 1044D; # DESERET CAPITAL LETTER ENG
diff --git a/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt b/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt
@@ -0,0 +1,1202 @@
+# CaseFolding-5.2.0.txt
+# Date: 2009-05-28, 23:02:34 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2009 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#223;e" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding 
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard, Version 5.0.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+# @missing 0000..10FFFF; <codepoint>
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE
+023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE
+023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR
+023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
+0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP
+0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE
+0244; C; 0289; # LATIN CAPITAL LETTER U BAR
+0245; C; 028C; # LATIN CAPITAL LETTER TURNED V
+0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE
+0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE
+024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
+024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE
+024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0370; C; 0371; # GREEK CAPITAL LETTER HETA
+0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI
+0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+03F7; C; 03F8; # GREEK CAPITAL LETTER SHO
+03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL
+03FA; C; 03FB; # GREEK CAPITAL LETTER SAN
+03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
+03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
+03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
+04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK
+04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE
+0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK
+0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA
+0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA
+0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE
+051A; C; 051B; # CYRILLIC CAPITAL LETTER QA
+051C; C; 051D; # CYRILLIC CAPITAL LETTER WE
+051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA
+0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
+0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
+0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN
+10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN
+10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN
+10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON
+10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN
+10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN
+10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN
+10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN
+10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN
+10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN
+10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS
+10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN
+10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR
+10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON
+10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR
+10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR
+10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE
+10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN
+10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR
+10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN
+10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR
+10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR
+10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN
+10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR
+10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN
+10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN
+10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN
+10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL
+10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL
+10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR
+10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN
+10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN
+10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE
+10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE
+10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE
+10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE
+10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR
+10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S
+1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL
+1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V
+1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2132; C; 214E; # TURNED CAPITAL F
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N

From pypy.commits at gmail.com  Wed Aug 24 05:41:54 2016
From: pypy.commits at gmail.com (marky1991)
Date: Wed, 24 Aug 2016 02:41:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merging with upstream 3k
Message-ID: <57bd6be2.ca11c30a.3a3c4.8f23@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k
Changeset: r86468:159ee4a5dd97
Date: 2016-08-23 17:46 -0400
http://bitbucket.org/pypy/pypy/changeset/159ee4a5dd97/

Log:	Merging with upstream 3k

diff too long, truncating to 2000 out of 35449 lines

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -85,10 +85,11 @@
     pass
 
 def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')):
+    import ast
     result = []
     for word in re_word.findall(line):
         if word.startswith('"'):
-            word = eval(word)
+            word = ast.literal_eval(word)
         result.append(word)
     return result
 
diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib-python/2.7/test/test_hash.py b/lib-python/2.7/test/test_hash.py
--- a/lib-python/2.7/test/test_hash.py
+++ b/lib-python/2.7/test/test_hash.py
@@ -174,7 +174,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 
diff --git a/lib-python/3/test/test_compileall.py b/lib-python/3/test/test_compileall.py
--- a/lib-python/3/test/test_compileall.py
+++ b/lib-python/3/test/test_compileall.py
@@ -202,11 +202,10 @@
 
     # Ensure that the default behavior of compileall's CLI is to create
     # PEP 3147 pyc/pyo files.
-    _pyo = 'pyo' if support.check_impl_detail() else 'pyc'
     for name, ext, switch in [
         ('normal', 'pyc', []),
-        ('optimize', _pyo, ['-O']),
-        ('doubleoptimize', _pyo, ['-OO']),
+        ('optimize', 'pyo', ['-O']),
+        ('doubleoptimize', 'pyo', ['-OO']),
     ]:
         def f(self, ext=ext, switch=switch):
             script_helper.assert_python_ok(*(switch +
diff --git a/lib-python/3/test/test_hash.py b/lib-python/3/test/test_hash.py
--- a/lib-python/3/test/test_hash.py
+++ b/lib-python/3/test/test_hash.py
@@ -164,7 +164,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 
diff --git a/lib-python/3/test/test_unicode.py b/lib-python/3/test/test_unicode.py
--- a/lib-python/3/test/test_unicode.py
+++ b/lib-python/3/test/test_unicode.py
@@ -2199,7 +2199,8 @@
     def test_getnewargs(self):
         text = 'abc'
         args = text.__getnewargs__()
-        self.assertIsNot(args[0], text)
+        if support.check_impl_detail():
+            self.assertIsNot(args[0], text)
         self.assertEqual(args[0], text)
         self.assertEqual(len(args), 1)
 
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -198,10 +198,13 @@
     return tp._alignmentofinstances()
 
 @builtinify
-def byref(cdata):
+def byref(cdata, offset=0):
     # "pointer" is imported at the end of this module to avoid circular
     # imports
-    return pointer(cdata)
+    ptr = pointer(cdata)
+    if offset != 0:
+        ptr._buffer[0] += offset
+    return ptr
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -0,0 +1,91 @@
+# Note: uses the CFFI out-of-line ABI mode.  We can't use the API
+# mode because ffi.compile() needs to run the compiler, which
+# needs 'subprocess', which needs 'msvcrt' and '_subprocess',
+# which depend on '_pypy_winbase_cffi' already.
+#
+# Note that if you need to regenerate _pypy_winbase_cffi and
+# can't use a preexisting PyPy to do that, then running this
+# file should work as long as 'subprocess' is not imported
+# by cffi.  I had to hack in 'cffi._pycparser' to move an
+#'import subprocess' to the inside of a function.  (Also,
+# CPython+CFFI should work as well.)
+#
+# This module supports both msvcrt.py and _subprocess.py.
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pypy_winbase_cffi", None)
+
+# ---------- MSVCRT ----------
+
+ffi.cdef("""
+typedef unsigned short wint_t;
+
+int _open_osfhandle(intptr_t osfhandle, int flags);
+intptr_t _get_osfhandle(int fd);
+int _setmode(int fd, int mode);
+int _locking(int fd, int mode, long nbytes);
+
+int _kbhit(void);
+int _getch(void);
+wint_t _getwch(void);
+int _getche(void);
+wint_t _getwche(void);
+int _putch(int);
+wint_t _putwch(wchar_t);
+int _ungetch(int);
+wint_t _ungetwch(wint_t);
+""")
+
+# ---------- SUBPROCESS ----------
+
+ffi.cdef("""
+typedef struct {
+    DWORD  cb;
+    char * lpReserved;
+    char * lpDesktop;
+    char * lpTitle;
+    DWORD  dwX;
+    DWORD  dwY;
+    DWORD  dwXSize;
+    DWORD  dwYSize;
+    DWORD  dwXCountChars;
+    DWORD  dwYCountChars;
+    DWORD  dwFillAttribute;
+    DWORD  dwFlags;
+    WORD   wShowWindow;
+    WORD   cbReserved2;
+    LPBYTE lpReserved2;
+    HANDLE hStdInput;
+    HANDLE hStdOutput;
+    HANDLE hStdError;
+} STARTUPINFO, *LPSTARTUPINFO;
+
+typedef struct {
+    HANDLE hProcess;
+    HANDLE hThread;
+    DWORD  dwProcessId;
+    DWORD  dwThreadId;
+} PROCESS_INFORMATION, *LPPROCESS_INFORMATION;
+
+DWORD WINAPI GetVersion(void);
+BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD);
+BOOL WINAPI CloseHandle(HANDLE);
+HANDLE WINAPI GetCurrentProcess(void);
+BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
+                            DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
+BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
+BOOL WINAPI TerminateProcess(HANDLE, UINT);
+HANDLE WINAPI GetStdHandle(DWORD);
+""")
+
+# --------------------
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -0,0 +1,10 @@
+# auto-generated file
+import _cffi_backend
+
+ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
+    _version = 0x2601,
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+)
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -10,152 +10,99 @@
 
 # Declare external Win32 functions
 
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
+from _pypy_winbase_cffi import ffi as _ffi
+_kernel32 = _ffi.dlopen('kernel32')
 
 GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
 
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
 
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
+# Now the _subprocess module implementation
 
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
+def _WinError():
+    code, message = _ffi.getwinerror()
+    raise WindowsError(code, message)
 
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
+_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
 
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-_GetModuleFileNameW = _kernel32.GetModuleFileNameW
-_GetModuleFileNameW.argtypes = [ctypes.c_int, ctypes.c_wchar_p, ctypes.c_uint]
-_GetModuleFileNameW.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessW
-_CreateProcess.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_wchar_p, ctypes.c_wchar_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
-
-# Now the _winapi module implementation
-
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
-
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
+class _handle(object):
+    def __init__(self, c_handle):
+        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
+        self.c_handle = c_handle
+        if int(self) != -1:
+            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
 
     def __int__(self):
-        return self.handle
+        return int(_ffi.cast("intptr_t", self.c_handle))
 
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
+    def __repr__(self):
+        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
 
     def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
+        h = int(self)
+        if h != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+        return h
 
     def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
+        if int(self) != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+            _kernel32.CloseHandle(c_handle)
 
 def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
+    handles = _ffi.new("HANDLE[2]")
 
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
+    res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size)
 
     if not res:
         raise _WinError()
 
-    return _handle(read.value), _handle(write.value)
+    return _handle(handles[0]), _handle(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
+    return _handle(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
+    # CPython: the first three arguments are expected to be integers
+    target = _ffi.new("HANDLE[1]")
 
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
+    res = _kernel32.DuplicateHandle(
+        _ffi.cast("HANDLE", source_process),
+        _ffi.cast("HANDLE", source),
+        _ffi.cast("HANDLE", target_process),
+        target, access, inherit, options)
 
     if not res:
         raise _WinError()
 
-    return _handle(target.value)
+    return _handle(target[0])
+
+def _z(input):
+    if input is None:
+        return _ffi.NULL
+    if isinstance(input, basestring):
+        return str(input)
+    raise TypeError("string/unicode/None expected, got %r" % (
+        type(input).__name__,))
 
 def CreateProcess(name, command_line, process_attr, thread_attr,
                   inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
+    si = _ffi.new("STARTUPINFO *")
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
+        # CPython: these three handles are expected to be _handle objects
         if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
+            si.hStdInput = startup_info.hStdInput.c_handle
         if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
+            si.hStdOutput = startup_info.hStdOutput.c_handle
         if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
+            si.hStdError = startup_info.hStdError.c_handle
 
-    pi = _PROCESS_INFORMATION()
+    pi = _ffi.new("PROCESS_INFORMATION *")
     flags |= CREATE_UNICODE_ENVIRONMENT
 
     if env is not None:
@@ -164,47 +111,55 @@
             envbuf += "%s=%s\0" % (k, v)
         envbuf += '\0'
     else:
-        envbuf = None
+        envbuf = _ffi.NULL
 
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
+    res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL,
+                                   _ffi.NULL, inherit, flags, envbuf,
+                                   _z(start_dir), si, pi)
 
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
 
 def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
+    # CPython: the first argument is expected to be an integer.
+    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
+                                        milliseconds)
     if res < 0:
         raise _WinError()
 
     return res
 
 def GetExitCodeProcess(handle):
-    code = _c_int()
+    # CPython: the first argument is expected to be an integer.
+    code = _ffi.new("DWORD[1]")
 
-    res = _GetExitCodeProcess(int(handle), _byref(code))
+    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
 
     if not res:
         raise _WinError()
 
-    return code.value
+    return code[0]
 
 def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
+    # CPython: the first argument is expected to be an integer.
+    # The second argument is silently wrapped in a UINT.
+    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+                                     _ffi.cast("UINT", exitcode))
 
     if not res:
         raise _WinError()
 
 def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
+    stdhandle = _ffi.cast("DWORD", stdhandle)
+    res = _kernel32.GetStdHandle(stdhandle)
 
     if not res:
         return None
     else:
-        return res
+        # note: returns integer, not handle object
+        return int(_ffi.cast("intptr_t", res))
 
 def CloseHandle(handle):
     res = _CloseHandle(handle)
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py
--- a/lib_pypy/cffi/_pycparser/__init__.py
+++ b/lib_pypy/cffi/_pycparser/__init__.py
@@ -10,7 +10,6 @@
 __all__ = ['c_lexer', 'c_parser', 'c_ast']
 __version__ = '2.14'
 
-from subprocess import Popen, PIPE
 from .c_parser import CParser
 
 
@@ -28,6 +27,7 @@
         When successful, returns the preprocessed file's contents.
         Errors from cpp will be printed out.
     """
+    from subprocess import Popen, PIPE
     path_list = [cpp_path]
     if isinstance(cpp_args, list):
         path_list += cpp_args
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -513,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -570,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -7,26 +7,39 @@
 # XXX incomplete: implemented only functions needed by subprocess.py
 # PAC: 2010/08 added MS locking for Whoosh
 
-import ctypes
+# 07/2016: rewrote in CFFI
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The 'msvcrt' module is only available on Windows")
+
+import _rawffi
+from _pypy_winbase_cffi import ffi as _ffi
+_lib = _ffi.dlopen(_rawffi.get_libc().name)
+
 import errno
-from ctypes_support import standard_c_lib as _c
-from ctypes_support import get_errno
-
-try:
-    open_osfhandle = _c._open_osfhandle
-except AttributeError: # we are not on windows
-    raise ImportError
 
 try: from __pypy__ import builtinify, validate_fd
 except ImportError: builtinify = validate_fd = lambda f: f
 
 
-open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int]
-open_osfhandle.restype = ctypes.c_int
+def _ioerr():
+    e = _ffi.errno
+    raise IOError(e, errno.errorcode[e])
 
-_get_osfhandle = _c._get_osfhandle
-_get_osfhandle.argtypes = [ctypes.c_int]
-_get_osfhandle.restype = ctypes.c_int
+
+ at builtinify
+def open_osfhandle(fd, flags):
+    """"open_osfhandle(handle, flags) -> file descriptor
+
+    Create a C runtime file descriptor from the file handle handle. The
+    flags parameter should be a bitwise OR of os.O_APPEND, os.O_RDONLY,
+    and os.O_TEXT. The returned file descriptor may be used as a parameter
+    to os.fdopen() to create a file object."""
+    fd = _lib._open_osfhandle(fd, flags)
+    if fd == -1:
+        _ioerr()
+    return fd
 
 @builtinify
 def get_osfhandle(fd):
@@ -38,62 +51,74 @@
         validate_fd(fd)
     except OSError as e:
         raise IOError(*e.args)
-    return _get_osfhandle(fd)
+    result = _lib._get_osfhandle(fd)
+    if result == -1:
+        _ioerr()
+    return result
 
-setmode = _c._setmode
-setmode.argtypes = [ctypes.c_int, ctypes.c_int]
-setmode.restype = ctypes.c_int
+ at builtinify
+def setmode(fd, flags):
+    """setmode(fd, mode) -> Previous mode
+
+    Set the line-end translation mode for the file descriptor fd. To set
+    it to text mode, flags should be os.O_TEXT; for binary, it should be
+    os.O_BINARY."""
+    flags = _lib._setmode(fd, flags)
+    if flags == -1:
+        _ioerr()
+    return flags
 
 LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5)
 
-_locking = _c._locking
-_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_locking.restype = ctypes.c_int
-
 @builtinify
 def locking(fd, mode, nbytes):
-    '''lock or unlock a number of bytes in a file.'''
-    rv = _locking(fd, mode, nbytes)
+    """"locking(fd, mode, nbytes) -> None
+
+    Lock part of a file based on file descriptor fd from the C runtime.
+    Raises IOError on failure. The locked region of the file extends from
+    the current file position for nbytes bytes, and may continue beyond
+    the end of the file. mode must be one of the LK_* constants listed
+    below. Multiple regions in a file may be locked at the same time, but
+    may not overlap. Adjacent regions are not merged; they must be unlocked
+    individually."""
+    rv = _lib._locking(fd, mode, nbytes)
     if rv != 0:
-        e = get_errno()
-        raise IOError(e, errno.errorcode[e])
+        _ioerr()
 
 # Console I/O routines
 
-kbhit = _c._kbhit
-kbhit.argtypes = []
-kbhit.restype = ctypes.c_int
+kbhit = _lib._kbhit
 
-getch = _c._getch
-getch.argtypes = []
-getch.restype = ctypes.c_char
+ at builtinify
+def getch():
+    return chr(_lib._getch())
 
-getwch = _c._getwch
-getwch.argtypes = []
-getwch.restype = ctypes.c_wchar
+ at builtinify
+def getwch():
+    return unichr(_lib._getwch())
 
-getche = _c._getche
-getche.argtypes = []
-getche.restype = ctypes.c_char
+ at builtinify
+def getche():
+    return chr(_lib._getche())
 
-getwche = _c._getwche
-getwche.argtypes = []
-getwche.restype = ctypes.c_wchar
+ at builtinify
+def getwche():
+    return unichr(_lib._getwche())
 
-putch = _c._putch
-putch.argtypes = [ctypes.c_char]
-putch.restype = None
+ at builtinify
+def putch(ch):
+    _lib._putch(ord(ch))
 
-putwch = _c._putwch
-putwch.argtypes = [ctypes.c_wchar]
-putwch.restype = None
+ at builtinify
+def putwch(ch):
+    _lib._putwch(ord(ch))
 
-ungetch = _c._ungetch
-ungetch.argtypes = [ctypes.c_char]
-ungetch.restype = None
+ at builtinify
+def ungetch(ch):
+    if _lib._ungetch(ord(ch)) == -1:   # EOF
+        _ioerr()
 
-ungetwch = _c._ungetwch
-ungetwch.argtypes = [ctypes.c_wchar]
-ungetwch.restype = None
-
-del ctypes
+ at builtinify
+def ungetwch(ch):
+    if _lib._ungetwch(ord(ch)) == -1:   # EOF
+        _ioerr()
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -40,7 +40,7 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
     "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy"
-    "faulthandler",
+    "faulthandler", "_jitlog",
 ])
 
 from rpython.jit.backend import detect_cpu
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -104,27 +104,24 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev liblzma-dev
-
-For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+    tk-dev libgc-dev \
+    liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel
-
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+    gdbm-devel \
+    xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses
+    libexpat-devel libffi-devel python-curses \
+    xz-devel  # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
-
 On Mac OS X, most of these build-time dependencies are installed alongside
 the Developer Tools. However, note that in order for the installation to
 find them you may need to run::
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -335,3 +335,65 @@
 
 This will disable SELinux's protection and allow PyPy to configure correctly.
 Be sure to enable it again if you need it!
+
+
+How should I report a bug?
+--------------------------
+
+Our bug tracker is here: https://bitbucket.org/pypy/pypy/issues/
+
+Missing features or incompatibilities with CPython are considered
+bugs, and they are welcome.  (See also our list of `known
+incompatibilities`__.)
+
+.. __: http://pypy.org/compat.html
+
+For bugs of the kind "I'm getting a PyPy crash or a strange
+exception", please note that: **We can't do anything without
+reproducing the bug ourselves**.  We cannot do anything with
+tracebacks from gdb, or core dumps.  This is not only because the
+standard PyPy is compiled without debug symbols.  The real reason is
+that a C-level traceback is usually of no help at all in PyPy.
+Debugging PyPy can be annoying.
+
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
+In more details:
+
+* First, please give the exact PyPy version, and the OS.
+
+* It might help focus our search if we know if the bug can be
+  reproduced on a "``pypy --jit off``" or not.  If "``pypy --jit
+  off``" always works, then the problem might be in the JIT.
+  Otherwise, we know we can ignore that part.
+
+* If you got the bug using only Open Source components, please give a
+  step-by-step guide that we can follow to reproduce the problem
+  ourselves.  Don't assume we know anything about any program other
+  than PyPy.  We would like a guide that we can follow point by point
+  (without guessing or having to figure things out)
+  on a machine similar to yours, starting from a bare PyPy, until we
+  see the same problem.  (If you can, you can try to reduce the number
+  of steps and the time it needs to run, but that is not mandatory.)
+
+* If the bug involves Closed Source components, or just too many Open
+  Source components to install them all ourselves, then maybe you can
+  give us some temporary ssh access to a machine where the bug can be
+  reproduced.  Or, maybe we can download a VirtualBox or VMWare
+  virtual machine where the problem occurs.
+
+* If giving us access would require us to use tools other than ssh,
+  make appointments, or sign a NDA, then we can consider a commerical
+  support contract for a small sum of money.
+
+* If even that is not possible for you, then sorry, we can't help.
+
+Of course, you can try to debug the problem yourself, and we can help
+you get started if you ask on the #pypy IRC channel, but be prepared:
+debugging an annoying PyPy problem usually involves quite a lot of gdb
+in auto-generated C code, and at least some knowledge about the
+various components involved, from PyPy's own RPython source code to
+the GC and possibly the JIT.
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -39,17 +39,16 @@
 library.
 
 If you want to install 3rd party libraries, the most convenient way is
-to install pip_ (unless you want to install virtualenv as explained
-below; then you can directly use pip inside virtualenvs):
+to install pip_ using ensurepip_ (unless you want to install virtualenv as 
+explained below; then you can directly use pip inside virtualenvs):
 
 .. code-block:: console
 
-    $ curl -O https://bootstrap.pypa.io/get-pip.py
-    $ ./pypy-2.1/bin/pypy get-pip.py
-    $ ./pypy-2.1/bin/pip install pygments  # for example
+    $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install pygments  # for example
 
-Third party libraries will be installed in ``pypy-2.1/site-packages``, and
-the scripts in ``pypy-2.1/bin``.
+Third party libraries will be installed in ``pypy-xxx/site-packages``, and
+the scripts in ``pypy-xxx/bin``.
 
 
 Installing using virtualenv
@@ -61,7 +60,7 @@
 checkout::
 
 	# from a tarball
-	$ virtualenv -p /opt/pypy-c-jit-41718-3fb486695f20-linux/bin/pypy my-pypy-env
+	$ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env
 
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
@@ -69,7 +68,7 @@
 Note that bin/python is now a symlink to bin/pypy.
 
 .. _pip: http://pypi.python.org/pypi/pip
-
+.. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html
 
 Building PyPy yourself
 ~~~~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -53,3 +53,100 @@
 
 Refactor PyTupleObject to look like cpython's and allow subclassing 
 PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -389,7 +389,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -408,8 +409,11 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            assert depth >= 0
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -409,25 +409,6 @@
         return ast.Try(body, handlers, otherwise, finally_suite,
                        try_node.get_lineno(), try_node.get_column())
 
-    def handle_with_stmt(self, with_node):
-        body = self.handle_suite(with_node.get_child(-1))
-        i = with_node.num_children() - 1
-        while True:
-            i -= 2
-            item = with_node.get_child(i)
-            test = self.handle_expr(item.get_child(0))
-            if item.num_children() == 3:
-                target = self.handle_expr(item.get_child(2))
-                self.set_context(target, ast.Store)
-            else:
-                target = None
-            wi = ast.With(test, target, body, with_node.get_lineno(),
-                          with_node.get_column())
-            if i == 1:
-                break
-            body = [wi]
-        return wi
-
     def handle_with_item(self, item_node):
         test = self.handle_expr(item_node.get_child(0))
         if item_node.num_children() == 3:
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -114,8 +114,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, int) and isinstance(right, int):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1307,3 +1307,22 @@
             counts = self.count_instructions(source)
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
+
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1748,6 +1748,23 @@
                 "Python int too large for C unsigned short")
         return value
 
+    def c_uid_t_w(self, w_obj):
+        # xxx assumes that uid_t and gid_t are a C unsigned int.
+        # Equivalent to space.c_uint_w(), with the exception that
+        # it also accepts -1 and converts that to UINT_MAX, which
+        # is (uid_t)-1.  And values smaller than -1 raise
+        # OverflowError, not ValueError.
+        try:
+            return self.c_uint_w(w_obj)
+        except OperationError as e:
+            if e.match(self, self.w_ValueError):
+                # ValueError: cannot convert negative integer to unsigned
+                if self.int_w(w_obj) == -1:
+                    return UINT_MAX
+                raise oefmt(self.w_OverflowError,
+                            "user/group id smaller than minimum (-1)")
+            raise
+
     def truncatedint_w(self, w_obj, allow_conversion=True):
         # Like space.gateway_int_w(), but return the integer truncated
         # instead of raising OverflowError.  For obscure cases only.
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -40,9 +40,6 @@
         and possibly more locals."""
         return self.signature().getallvarnames()
 
-    def getformalargcount(self):
-        return self.signature().scope_length()
-
     def getdocstring(self, space):
         return space.w_None
 
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -38,7 +38,9 @@
                           'name?',
                           'w_kw_defs?']
 
-    def __init__(self, space, code, w_globals=None, defs_w=[], w_kw_defs=None,
+    w_kw_defs = None
+
+    def __init__(self, space, code, w_globals=None, defs_w=[], kw_defs_w=None,
                  closure=None, w_ann=None, forcename=None, qualname=None):
         self.space = space
         self.name = forcename or code.co_name
@@ -48,10 +50,12 @@
         self.w_func_globals = w_globals  # the globals dictionary
         self.closure = closure    # normally, list of Cell instances or None
         self.defs_w = defs_w
-        self.w_kw_defs = w_kw_defs
         self.w_func_dict = None # filled out below if needed
         self.w_module = None
         self.w_ann = w_ann
+        #
+        if kw_defs_w is not None:
+            self.init_kwdefaults_dict(kw_defs_w)
 
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
@@ -379,14 +383,23 @@
 
     def fset_func_kwdefaults(self, space, w_new):
         if space.is_w(w_new, space.w_None):
-            w_new = None
-        elif not space.isinstance_w(w_new, space.w_dict):
-            raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
-        self.w_kw_defs = w_new
+            self.w_kw_defs = None
+        else:
+            if not space.isinstance_w(w_new, space.w_dict):
+                raise oefmt(space.w_TypeError, "__kwdefaults__ must be a dict")
+            self.w_kw_defs = w_new
 
     def fdel_func_kwdefaults(self, space):
         self.w_kw_defs = None
 
+    def init_kwdefaults_dict(self, kw_defs_w):
+        # use the moduledict logic to get normally-constant entries
+        space = self.space
+        w_dict = space.newdict(module=True)
+        for w_name, w_value in kw_defs_w:
+            space.setitem(w_dict, w_name, w_value)
+        self.w_kw_defs = w_dict
+
     def fget_func_doc(self, space):
         if self.w_doc is None:
             self.w_doc = self.code.getdocstring(space)
@@ -663,10 +676,12 @@
     def __init__(self, func):
         assert isinstance(func, Function)
         Function.__init__(self, func.space, func.code, func.w_func_globals,
-                          func.defs_w, None, func.closure, None, func.name)
+                          func.defs_w, None, func.closure,
+                          None, func.name)
         self.w_doc = func.w_doc
         self.w_func_dict = func.w_func_dict
         self.w_module = func.w_module
+        self.w_kw_defs = func.w_kw_defs
 
     def descr_builtinfunction__new__(space, w_subtype):
         raise oefmt(space.w_TypeError,
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -28,6 +28,8 @@
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
 
+NO_DEFAULT = object()
+
 
 # internal non-translatable parts:
 class SignatureBuilder(object):
@@ -44,12 +46,21 @@
         self.argnames = argnames
         self.varargname = varargname
         self.kwargname = kwargname
+        self.kwonlyargnames = None
 
     def append(self, argname):
-        self.argnames.append(argname)
+        if self.kwonlyargnames is None:
+            self.argnames.append(argname)
+        else:
+            self.kwonlyargnames.append(argname)
+
+    def marker_kwonly(self):
+        assert self.kwonlyargnames is None
+        self.kwonlyargnames = []
 
     def signature(self):
-        return Signature(self.argnames, self.varargname, self.kwargname)
+        return Signature(self.argnames, self.varargname, self.kwargname,
+                         self.kwonlyargnames)
 
 #________________________________________________________________
 
@@ -66,13 +77,6 @@
         """NOT_RPYTHON"""
         raise NotImplementedError
 
-def kwonly(arg_unwrapper):
-    """Mark argument as keyword-only.
-
-    XXX: has no actual effect for now.
-    """
-    return arg_unwrapper
-
 
 class UnwrapSpecRecipe(object):
     "NOT_RPYTHON"
@@ -177,6 +181,9 @@
     def visit_c_ushort(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_c_uid_t(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_truncatedint_w(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -226,6 +233,11 @@
         name = int_unwrapping_space_method(typ)
         self.checked_space_method(name, app_sig)
 
+    def visit_kwonly(self, _, app_sig):
+        argname = self.orig_arg()
+        assert argname == '__kwonly__'
+        app_sig.marker_kwonly()
+
 
 class UnwrapSpec_EmitRun(UnwrapSpecEmit):
 
@@ -307,9 +319,15 @@
     def visit_c_ushort(self, typ):
         self.run_args.append("space.c_ushort_w(%s)" % (self.scopenext(),))
 
+    def visit_c_uid_t(self, typ):
+        self.run_args.append("space.c_uid_t_w(%s)" % (self.scopenext(),))
+
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
+    def visit_kwonly(self, typ):
+        self.run_args.append("None")
+
     def _make_unwrap_activation_class(self, unwrap_spec, cache={}):
         try:
             key = tuple(unwrap_spec)
@@ -456,9 +474,15 @@
     def visit_c_ushort(self, typ):
         self.unwrap.append("space.c_ushort_w(%s)" % (self.nextarg(),))
 
+    def visit_c_uid_t(self, typ):
+        self.unwrap.append("space.c_uid_t_w(%s)" % (self.nextarg(),))
+
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
+    def visit_kwonly(self, typ):
+        raise FastFuncNotSupported
+
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
         unwrap_info.apply_over(unwrap_spec)
@@ -554,6 +578,8 @@
                 unwrap_spec.append('args_w')
             elif argname.startswith('w_'):
                 unwrap_spec.append(W_Root)
+            elif argname == '__kwonly__':
+                unwrap_spec.append('kwonly')
             else:
                 unwrap_spec.append(None)
 
@@ -603,7 +629,12 @@
 
         # First extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(func.func_code)
+        sig = pycode.cpython_code_signature(func.func_code)
+        argnames = sig.argnames
+        varargname = sig.varargname
+        kwargname = sig.kwargname
+        if sig.kwonlyargnames:
+            import pdb; pdb.set_trace()
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -627,7 +658,9 @@
         app_sig = SignatureBuilder(func)
 
         UnwrapSpec_Check(orig_sig).apply_over(unwrap_spec, app_sig)
-        self.sig = argnames, varargname, kwargname = app_sig.signature()
+        self.sig = app_sig.signature()
+        argnames = self.sig.argnames
+        varargname = self.sig.varargname
 
         self.minargs = len(argnames)
         if varargname:
@@ -936,64 +969,71 @@
         self.name = app_name
         self.as_classmethod = as_classmethod
 
-        if not f.func_defaults:
-            self._staticdefs = []
-        else:
-            argnames = self._code._argnames
-            defaults = f.func_defaults
-            self._staticdefs = zip(argnames[-len(defaults):], defaults)
+        argnames = self._code._argnames
+        defaults = f.func_defaults or ()
+        self._staticdefs = dict(zip(
+            argnames[len(argnames) - len(defaults):], defaults))
+
         return self
 
     def _getdefaults(self, space):
         "NOT_RPYTHON"
-        defs_w = []
-        unwrap_spec = self._code._unwrap_spec[-len(self._staticdefs):]
-        for i, (name, defaultval) in enumerate(self._staticdefs):
+        alldefs_w = {}
+        assert len(self._code._argnames) == len(self._code._unwrap_spec)
+        for name, spec in zip(self._code._argnames, self._code._unwrap_spec):
+            if name == '__kwonly__':
+                continue
+
+            defaultval = self._staticdefs.get(name, NO_DEFAULT)
+            w_def = Ellipsis
             if name.startswith('w_'):
-                assert defaultval is None, (
+                assert defaultval in (NO_DEFAULT, None), (
                     "%s: default value for '%s' can only be None, got %r; "
                     "use unwrap_spec(...=WrappedDefault(default))" % (
                     self._code.identifier, name, defaultval))
-                defs_w.append(None)
-            elif name != '__args__' and name != 'args_w':
-                spec = unwrap_spec[i]
-                if isinstance(defaultval, str) and spec not in [str]:
-                    defs_w.append(space.newbytes(defaultval))
-                else:
-                    defs_w.append(space.wrap(defaultval))
-        if self._code._unwrap_spec:
-            UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig[0])
-            if defs_w:
-                alldefs_w[-len(defs_w):] = defs_w
-            code = self._code
-            assert isinstance(code._unwrap_spec, (list, tuple))
-            assert isinstance(code._argnames, list)
-            assert len(code._unwrap_spec) == len(code._argnames)
-            for i in range(len(code._unwrap_spec)-1, -1, -1):
-                spec = code._unwrap_spec[i]
-                argname = code._argnames[i]
-                if isinstance(spec, tuple) and spec[0] is W_Root:
-                    assert False, "use WrappedDefault"
-                if isinstance(spec, WrappedDefault):
-                    default_value = spec.default_value
-                    if isinstance(default_value, str):
-                        w_default = space.newbytes(default_value)
-                    else:
-                        w_default = space.wrap(default_value)
-                    assert isinstance(w_default, W_Root)
-                    assert argname.startswith('w_')
-                    argname = argname[2:]
-                    j = self._code.sig[0].index(argname)
-                    assert alldefs_w[j] in (UNDEFINED, None)
-                    alldefs_w[j] = w_default
-            first_defined = 0
-            while (first_defined < len(alldefs_w) and
-                   alldefs_w[first_defined] is UNDEFINED):
-                first_defined += 1
-            defs_w = alldefs_w[first_defined:]
-            assert UNDEFINED not in defs_w
-        return defs_w
+                if defaultval is None:
+                    w_def = None
+
+            if isinstance(spec, tuple) and spec[0] is W_Root:
+                assert False, "use WrappedDefault"
+            elif isinstance(spec, WrappedDefault):
+                assert name.startswith('w_')
+                defaultval = spec.default_value
+                w_def = Ellipsis
+
+            if defaultval is not NO_DEFAULT:
+                if name != '__args__' and name != 'args_w':
+                    if w_def is Ellipsis:
+                        if isinstance(defaultval, str) and spec not in [str]:
+                            w_def = space.newbytes(defaultval)
+                        else:
+                            w_def = space.wrap(defaultval)
+                    if name.startswith('w_'):
+                        name = name[2:]
+                    alldefs_w[name] = w_def
+        #
+        # Here, 'alldefs_w' maps some argnames to their wrapped default
+        # value.  We return two lists:
+        #  - a list of defaults for positional arguments, which covers
+        #    some suffix of the sig.argnames list
+        #  - a list of pairs (w_name, w_def) for kwonly arguments
+        #
+        sig = self._code.sig
+        first_defined = 0
+        while (first_defined < len(sig.argnames) and
+               sig.argnames[first_defined] not in alldefs_w):
+            first_defined += 1
+        defs_w = [alldefs_w.pop(name) for name in sig.argnames[first_defined:]]
+
+        kw_defs_w = None
+        if alldefs_w:
+            kw_defs_w = []
+            for name, w_def in sorted(alldefs_w.items()):
+                assert name in sig.kwonlyargnames
+                w_name = space.newunicode(name.decode('utf-8'))
+                kw_defs_w.append((w_name, w_def))
+
+        return defs_w, kw_defs_w
 
     # lazy binding to space
 
@@ -1013,9 +1053,10 @@
     def build(cache, gateway):
         "NOT_RPYTHON"
         space = cache.space
-        defs = gateway._getdefaults(space) # needs to be implemented by subclass
+        defs_w, kw_defs_w = gateway._getdefaults(space)
         code = gateway._code
-        fn = FunctionWithFixedCode(space, code, None, defs, forcename=gateway.name)
+        fn = FunctionWithFixedCode(space, code, None, defs_w, kw_defs_w,
+                                   forcename=gateway.name)
         if not space.config.translating:
             fn.add_to_table()
         if gateway.as_classmethod:
@@ -1153,15 +1194,15 @@
             source = source[source.find('\n') + 1:].lstrip()
         assert source.startswith("def "), "can only transform functions"
         source = source[4:]
-        import __future__
-        if flags & __future__.CO_FUTURE_DIVISION:
-            prefix += "from __future__ import division\n"
-        if flags & __future__.CO_FUTURE_ABSOLUTE_IMPORT:
-            prefix += "from __future__ import absolute_import\n"
-        if flags & __future__.CO_FUTURE_PRINT_FUNCTION:
-            prefix += "from __future__ import print_function\n"
-        if flags & __future__.CO_FUTURE_UNICODE_LITERALS:
-            prefix += "from __future__ import unicode_literals\n"
+        # The following flags have no effect any more in app-level code
+        # (i.e. they are always on anyway), and have been removed:
+        #    CO_FUTURE_DIVISION
+        #    CO_FUTURE_ABSOLUTE_IMPORT
+        #    CO_FUTURE_PRINT_FUNCTION
+        #    CO_FUTURE_UNICODE_LITERALS
+        # Original code was, for each of these flags:
+        #    if flags & __future__.CO_xxx:
+        #        prefix += "from __future__ import yyy\n"
     p = source.find('(')
     assert p >= 0
     funcname = source[:p].strip()
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -19,6 +19,7 @@
         """ NOT_RPYTHON """
         Module.__init__(self, space, w_name)
         self.lazy = True
+        self.lazy_initial_values_w = {}
         self.__class__.buildloaders()
         self.loaders = self.loaders.copy()    # copy from the class to the inst
         self.submodules_w = []
@@ -57,22 +58,11 @@
             if not self.lazy and self.w_initialdict is None:
                 self.save_module_content_for_future_reload()
 
-    def save_module_content_for_future_reload(self, save_all=False):
-        # Because setdictvalue is unable to immediately load all attributes
-        # (due to an importlib bootstrapping problem), this method needs to be
-        # able to support saving the content of a module's dict without
-        # requiring that the entire dict already be loaded. To support that
-        # properly, when updating the dict, we must be careful to never
-        # overwrite the value of a key already in w_initialdict. (So as to avoid
-        # overriding the builtin value with a user-provided value)
-        if self.space.is_none(self.w_initialdict) or save_all:
-            self.w_initialdict = self.space.call_method(self.w_dict, 'copy')
-        else:
-            w_items = self.space.call_method(self.w_dict, 'items')
-            for w_item in self.space.iteriterable(w_items):
-                w_key, w_value = self.space.fixedview(w_item, expected_length=2)
-                if not self.space.contains_w(self.w_initialdict, w_key):
-                    self.space.setitem(self.w_initialdict, w_key, w_value)
+    def save_module_content_for_future_reload(self):
+        # Save the current dictionary in w_initialdict, for future
+        # reloads.  This forces the dictionary if needed.
+        w_dict = self.getdict(self.space)
+        self.w_initialdict = self.space.call_method(w_dict, 'copy')
 
     @classmethod
     def get_applevel_name(cls):
@@ -101,9 +91,13 @@
         return w_value
 
     def setdictvalue(self, space, attr, w_value):
-        if self.lazy:
-            self._load_lazily(space, attr)
-            self.save_module_content_for_future_reload()
+        if self.lazy and attr not in self.lazy_initial_values_w:
+            # in lazy mode, the first time an attribute changes,
+            # we save away the old (initial) value.  This allows
+            # a future getdict() call to build the correct
+            # self.w_initialdict, containing the initial value.
+            w_initial_value = self._load_lazily(space, attr)
+            self.lazy_initial_values_w[attr] = w_initial_value
         space.setitem_str(self.w_dict, attr, w_value)
         return True
 
@@ -137,13 +131,29 @@
 
     def getdict(self, space):
         if self.lazy:
-            for name in self.loaders:
-                w_value = self.get(name)
-                space.setitem(self.w_dict, space.new_interned_str(name), w_value)
-            self.lazy = False
-            self.save_module_content_for_future_reload()
+            self._force_lazy_dict_now()
         return self.w_dict
 
+    def _force_lazy_dict_now(self):
+        # Force the dictionary by calling all lazy loaders now.
+        # This also saves in self.w_initialdict a copy of all the
+        # initial values, including if they have already been
+        # modified by setdictvalue().
+        space = self.space
+        for name in self.loaders:
+            w_value = self.get(name)
+            space.setitem(self.w_dict, space.new_interned_str(name), w_value)
+        self.lazy = False
+        self.save_module_content_for_future_reload()
+        for key, w_initial_value in self.lazy_initial_values_w.items():
+            w_key = space.new_interned_str(key)
+            if w_initial_value is not None:
+                space.setitem(self.w_initialdict, w_key, w_initial_value)
+            else:
+                if space.finditem(self.w_initialdict, w_key) is not None:
+                    space.delitem(self.w_initialdict, w_key)
+        del self.lazy_initial_values_w
+
     def _cleanup_(self):
         self.getdict(self.space)
         self.w_initialdict = None
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -45,7 +45,7 @@
 
 # cpython_code_signature helper
 def cpython_code_signature(code):
-    "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
+    """Return a Signature instance."""
     argcount = code.co_argcount
     varnames = code.co_varnames
     if we_are_translated():
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1235,15 +1235,15 @@
             for i in range(len(names_w) - 1, -1, -1):
                 space.setitem(w_ann, names_w[i], self.popvalue())
         defaultarguments = self.popvalues(posdefaults)
-        w_kw_defs = None
+        kw_defs_w = None
         if kwdefaults:
-            w_kw_defs = space.newdict(strdict=True)
-            for i in range(kwdefaults - 1, -1, -1):
-                w_name = self.popvalue()
-                w_def = self.popvalue()
-                space.setitem(w_kw_defs, w_def, w_name)
+            kw_defs_w = []
+            for i in range(kwdefaults):
+                w_defvalue = self.popvalue()
+                w_defname = self.popvalue()
+                kw_defs_w.append((w_defname, w_defvalue))
         fn = function.Function(space, codeobj, self.get_w_globals(), defaultarguments,
-                               w_kw_defs, freevars, w_ann, qualname=qualname)
+                               kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
     def MAKE_FUNCTION(self, oparg, next_instr):
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -39,6 +39,7 @@
 
     def scope_length(self):
         scopelen = len(self.argnames)
+        scopelen += len(self.kwonlyargnames)
         scopelen += self.has_vararg()
         scopelen += self.has_kwarg()
         return scopelen
@@ -68,18 +69,3 @@
         if not isinstance(other, Signature):
             return NotImplemented
         return not self == other
-
-
-    # make it look tuply for its use in the annotator
-
-    def __len__(self):
-        return 3
-
-    def __getitem__(self, i):
-        if i == 0:
-            return self.argnames
-        if i == 1:
-            return self.varargname
-        if i == 2:
-            return self.kwargname
-        raise IndexError
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -209,7 +209,6 @@
         self.check(['-c', 'pass'], {'PYTHONNOUSERSITE': '1'}, sys_argv=['-c'],
                    run_command='pass', **expected)
 
-
 class TestInteraction:
     """
     These tests require pexpect (UNIX-only).
@@ -1152,4 +1151,3 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -30,7 +30,7 @@
         assert sig.num_argnames() == 3
         assert sig.has_vararg()
         assert sig.has_kwarg()
-        assert sig.scope_length() == 5
+        assert sig.scope_length() == 6
         assert sig.getallvarnames() == ["a", "b", "c", "d", "kwonly", "c"]
 
     def test_eq(self):
@@ -47,13 +47,6 @@
         assert sig.find_argname("d") == -1
         assert sig.find_argname("kwonly") == 3
 
-    def test_tuply(self):
-        sig = Signature(["a", "b", "c"], "d", "e")
-        x, y, z = sig
-        assert x == ["a", "b", "c"]
-        assert y == "d"
-        assert z == "e"
-
 class dummy_wrapped_dict(dict):
     def __nonzero__(self):
         raise NotImplementedError
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -47,6 +47,12 @@
         code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace, "index"])
         assert code.signature() == Signature(["index"], None, None)
 
+        def f(space, __kwonly__, w_x):
+            pass
+        code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace,
+                                                   "kwonly", W_Root])
+        assert code.signature() == Signature([], kwonlyargnames=['x'])
+
 
     def test_call(self):
         def c(space, w_x, w_y, hello_w):
@@ -753,7 +759,7 @@
         @gateway.unwrap_spec(w_x = WrappedDefault(42), y=int)
         def g(space, w_x, y):
             never_called
-        py.test.raises(AssertionError, space.wrap, gateway.interp2app_temp(g))
+        py.test.raises(KeyError, space.wrap, gateway.interp2app_temp(g))
 
     def test_unwrap_spec_default_applevel_bug2(self):
         space = self.space
@@ -803,6 +809,80 @@
         w_res = space.call_args(w_g, args)
         assert space.eq_w(w_res, space.newbytes('foo'))
 
+    def test_unwrap_spec_kwonly(self):
+        space = self.space
+        def g(space, w_x, __kwonly__, w_y):
+            return space.sub(w_x, w_y)
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w = space.wrap
+        w1 = w(1)
+
+        for i in range(4):
+            a = argument.Arguments(space, [w1, w1, w1])
+            py.test.raises(gateway.OperationError, space.call_args, w_g, a)
+            py.test.raises(gateway.OperationError, space.call_function, w_g,
+                           *(i * (w1,)))
+
+        args = argument.Arguments(space, [w(1)],
+                                  w_starstararg = w({'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-9))
+        args = argument.Arguments(space, [],
+                                  w_starstararg = w({'x': 2, 'y': 10}))
+        assert space.eq_w(space.call_args(w_g, args), w(-8))
+

From pypy.commits at gmail.com  Wed Aug 24 05:52:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 02:52:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix the test (failed on OS/X 64 because
 x==y==sys.maxint typically)
Message-ID: <57bd6e43.c186c20a.299e7.86e0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86470:6bfe423a36a4
Date: 2016-08-24 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/6bfe423a36a4/

Log:	Fix the test (failed on OS/X 64 because x==y==sys.maxint typically)

diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import sys
 
 import os
 if os.name != 'posix':
@@ -47,6 +48,9 @@
     # minimal "does not crash" test
     x, y = resource.getrlimit(resource.RLIMIT_CPU)
     resource.setrlimit(resource.RLIMIT_CPU, (x, y))
-    x += 0.2
-    y += 0.3
-    resource.setrlimit(resource.RLIMIT_CPU, (x, y))    # truncated to ints
+    # sometimes, x and y are very large (more than 53 bits).
+    # for these huge values, int(float(x)) > x...
+    xf = x + 0.2
+    yf = y + 0.3
+    if int(xf) == x and int(yf) == y:
+        resource.setrlimit(resource.RLIMIT_CPU, (x, y))  # truncated to ints

From pypy.commits at gmail.com  Wed Aug 24 05:57:56 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 02:57:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge redirect-assembler-jitlog
Message-ID: <57bd6fa4.d42f1c0a.d0cb9.c745@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r86471:4b3bf21d99c6
Date: 2016-08-24 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/4b3bf21d99c6/

Log:	merge redirect-assembler-jitlog

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1064,6 +1064,8 @@
         else:
             assert mc.get_relative_pos() <= 13
         mc.copy_to_raw_memory(oldadr)
+        # log the redirection of the call_assembler_* operation
+        jl.redirect_assembler(oldlooptoken, newlooptoken, target)
 
     def dump(self, text):
         if not self.verbose:
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 2
+JITLOG_VERSION = 3
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -244,6 +244,7 @@
     ('COMMON_PREFIX',),
     ('ABORT_TRACE',),
     ('SOURCE_CODE',),
+    ('REDIRECT_ASSEMBLER',),
 ]
 
 start = 0x11
@@ -309,6 +310,17 @@
     content = ''.join(list)
     jitlog_write_marked(content, len(content))
 
+def redirect_assembler(oldtoken, newtoken, target):
+    if not jitlog_enabled():
+        return
+    descr_nmr = compute_unique_id(oldtoken)
+    new_descr_nmr = compute_unique_id(newtoken)
+    list = [MARK_REDIRECT_ASSEMBLER, encode_le_addr(descr_nmr),
+            encode_le_addr(new_descr_nmr), encode_le_addr(target)]
+    content = ''.join(list)
+    jitlog_write_marked(content, len(content))
+
+
 class JitLogger(object):
     def __init__(self, cpu=None):
         self.cpu = cpu
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -6,6 +6,15 @@
 from rpython.jit.backend.model import AbstractCPU
 from rpython.jit.metainterp.history import ConstInt, ConstPtr
 from rpython.rlib.rjitlog import rjitlog as jl
+from rpython.jit.metainterp.history import AbstractDescr
+from rpython.rlib.objectmodel import compute_unique_id
+
+class FakeCallAssemblerLoopToken(AbstractDescr):
+    def __init__(self, target):
+	self._ll_function_addr = target
+
+    def repr_of_descr(self):
+        return 'looptoken'
 
 class FakeLog(object):
     def __init__(self):
@@ -109,3 +118,31 @@
         with py.test.raises(AssertionError):
             jl.commonprefix(None,None)
 
+    def test_redirect_assembler(self, tmpdir):
+        looptoken = FakeCallAssemblerLoopToken(0x0)
+        newlooptoken = FakeCallAssemblerLoopToken(0x1234)
+        #
+        logger = jl.JitLogger()
+        file = tmpdir.join('binary_file')
+        file.ensure()
+        fd = file.open('wb')
+        jl.jitlog_init(fd.fileno())
+        logger.start_new_trace(self.make_metainterp_sd(), jd_name='jdname')
+        log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
+        op = ResOperation(rop.CALL_ASSEMBLER_I, [], descr=looptoken)
+        log_trace.write([], [op])
+        jl.redirect_assembler(looptoken, newlooptoken, 0x1234)
+        #the next line will close 'fd', instead of logger.finish()
+        fd.close()
+        binary = file.read()
+        opnum = jl.encode_le_16bit(rop.CALL_ASSEMBLER_I)
+        id_looptoken = compute_unique_id(looptoken)
+        new_id_looptoken = compute_unique_id(newlooptoken)
+        end = jl.MARK_RESOP_DESCR + opnum + jl.encode_str('i0,looptoken') + \
+              jl.encode_le_addr(id_looptoken) + jl.encode_str('') + \
+              jl.MARK_REDIRECT_ASSEMBLER + \
+              jl.encode_le_addr(id_looptoken) + \
+              jl.encode_le_addr(new_id_looptoken) + \
+              jl.encode_le_addr(newlooptoken._ll_function_addr)
+        assert binary.endswith(end)
+        

From pypy.commits at gmail.com  Wed Aug 24 05:57:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 02:57:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <57bd6fa6.d4e01c0a.a8cdb.c518@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r86472:dbdfb99f8932
Date: 2016-08-24 11:55 +0200
http://bitbucket.org/pypy/pypy/changeset/dbdfb99f8932/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -155,3 +155,7 @@
 
 Support for memoryview attributes (format, itemsize, ...).
 Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.

From pypy.commits at gmail.com  Wed Aug 24 05:57:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 02:57:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: import issue
Message-ID: <57bd6fa7.e129c20a.4ccf.99cc@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86473:04a4bb69ff39
Date: 2016-08-24 11:12 +0200
http://bitbucket.org/pypy/pypy/changeset/04a4bb69ff39/

Log:	import issue

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -13,7 +13,6 @@
     WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
 from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std import newformat
-from pypy.objspace.std.basestringtype import basestring_typedef
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.unicodeobject import (

From pypy.commits at gmail.com  Wed Aug 24 05:58:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 02:58:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: buffer protocal c headers are stripped
 down in py3
Message-ID: <57bd6faa.12331c0a.eefe3.d676@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86474:d5bd69e6035c
Date: 2016-08-24 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/d5bd69e6035c/

Log:	buffer protocal c headers are stripped down in py3

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -11,7 +11,7 @@
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    getbufferproc, readbufferproc, ssizessizeobjargproc)
+    getbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -22,6 +22,9 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.sys.version import CPYTHON_VERSION
+
+PY3 = CPYTHON_VERSION[0] == 3
 
 # XXX: Also defined in object.h
 Py_LT = 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -14,7 +14,7 @@
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_buffer,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.lltypesystem.lltype import Ptr, FuncType, Void
 from pypy.module.cpyext.api import (cpython_struct, Py_ssize_t, Py_ssize_tP,
-    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_Buffer,
+    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_buffer,
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
@@ -54,10 +54,6 @@
 wrapperfunc = P(FT([PyO, PyO, rffi.VOIDP], PyO))
 wrapperfunc_kwds = P(FT([PyO, PyO, rffi.VOIDP, PyO], PyO))
 
-readbufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
-writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
-segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
-charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
 getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = P(FT([PyO, Py_bufferP], Void))
 
@@ -131,10 +127,6 @@
 ))
 
 PyBufferProcs = cpython_struct("PyBufferProcs", (
-    ("bf_getreadbuffer", readbufferproc),
-    ("bf_getwritebuffer", writebufferproc),
-    ("bf_getsegcount", segcountproc),
-    ("bf_getcharbuffer", charbufferproc),
     ("bf_getbuffer", getbufferproc),
     ("bf_releasebuffer", releasebufferproc),
 ))
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -15,9 +15,6 @@
 from pypy.objspace.std import newformat
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringmethods import StringMethods
-from pypy.objspace.std.unicodeobject import (
-    decode_object, unicode_from_encoded_object,
-    unicode_from_string, getdefaultencoding)
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 

From pypy.commits at gmail.com  Wed Aug 24 05:59:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 02:59:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge py3k
Message-ID: <57bd7017.531d1c0a.504c5.dc74@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86475:a43d6aa79232
Date: 2016-08-24 11:59 +0200
http://bitbucket.org/pypy/pypy/changeset/a43d6aa79232/

Log:	merge py3k

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -11,7 +11,7 @@
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    getbufferproc, readbufferproc, ssizessizeobjargproc)
+    getbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -22,6 +22,9 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.sys.version import CPYTHON_VERSION
+
+PY3 = CPYTHON_VERSION[0] == 3
 
 # XXX: Also defined in object.h
 Py_LT = 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -14,7 +14,7 @@
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_buffer,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.lltypesystem.lltype import Ptr, FuncType, Void
 from pypy.module.cpyext.api import (cpython_struct, Py_ssize_t, Py_ssize_tP,
-    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_Buffer,
+    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_buffer,
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
@@ -54,10 +54,6 @@
 wrapperfunc = P(FT([PyO, PyO, rffi.VOIDP], PyO))
 wrapperfunc_kwds = P(FT([PyO, PyO, rffi.VOIDP, PyO], PyO))
 
-readbufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
-writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
-segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
-charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
 getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = P(FT([PyO, Py_bufferP], Void))
 
@@ -131,10 +127,6 @@
 ))
 
 PyBufferProcs = cpython_struct("PyBufferProcs", (
-    ("bf_getreadbuffer", readbufferproc),
-    ("bf_getwritebuffer", writebufferproc),
-    ("bf_getsegcount", segcountproc),
-    ("bf_getcharbuffer", charbufferproc),
     ("bf_getbuffer", getbufferproc),
     ("bf_releasebuffer", releasebufferproc),
 ))
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -394,13 +394,6 @@
         of the specified width. The string S is never truncated.
         """
 
-    def descr_hex(self, space):
-        """S.hex() -> string
-
-        Creates a hexadecimal string of the bytes object
-        """
-
-
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -223,7 +223,7 @@
             pass
 
 if _WIN32:
-    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h']
+    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h']
     libraries = []
 else:
     if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')):
@@ -254,10 +254,10 @@
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
+    CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
-        CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),

From pypy.commits at gmail.com  Wed Aug 24 06:01:10 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 03:01:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: merge py3.5
Message-ID: <57bd7066.d42f1c0a.d0cb9.c8cf@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86476:9955cc83a3a8
Date: 2016-08-24 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/9955cc83a3a8/

Log:	merge py3.5

diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -49,9 +49,11 @@
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
-    # some parts of code.py are copied here because it seems to be impossible
+    # some parts of code.py are copied here because it was impossible
     # to start an interactive console without printing at least one line
-    # of banner
+    # of banner.  This was fixed in 3.4; but then from 3.6 it prints a
+    # line when exiting.  This can be disabled too---by passing an argument
+    # that doesn't exist in <= 3.5.  So, too much mess: just copy the code.
     more = 0
     while 1:
         try:
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -11,7 +11,7 @@
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    getbufferproc, readbufferproc, ssizessizeobjargproc)
+    getbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -22,6 +22,9 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.sys.version import CPYTHON_VERSION
+
+PY3 = CPYTHON_VERSION[0] == 3
 
 # XXX: Also defined in object.h
 Py_LT = 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -14,7 +14,7 @@
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_buffer,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.lltypesystem.lltype import Ptr, FuncType, Void
 from pypy.module.cpyext.api import (cpython_struct, Py_ssize_t, Py_ssize_tP,
-    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_Buffer,
+    PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_buffer,
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
@@ -54,10 +54,6 @@
 wrapperfunc = P(FT([PyO, PyO, rffi.VOIDP], PyO))
 wrapperfunc_kwds = P(FT([PyO, PyO, rffi.VOIDP, PyO], PyO))
 
-readbufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
-writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
-segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
-charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
 getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = P(FT([PyO, Py_bufferP], Void))
 
@@ -131,10 +127,6 @@
 ))
 
 PyBufferProcs = cpython_struct("PyBufferProcs", (
-    ("bf_getreadbuffer", readbufferproc),
-    ("bf_getwritebuffer", writebufferproc),
-    ("bf_getsegcount", segcountproc),
-    ("bf_getcharbuffer", charbufferproc),
     ("bf_getbuffer", getbufferproc),
     ("bf_releasebuffer", releasebufferproc),
 ))
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -394,13 +394,6 @@
         of the specified width. The string S is never truncated.
         """
 
-    def descr_hex(self, space):
-        """S.hex() -> string
-
-        Creates a hexadecimal string of the bytes object
-        """
-
-
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -223,7 +223,7 @@
             pass
 
 if _WIN32:
-    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h']
+    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h']
     libraries = []
 else:
     if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')):
@@ -254,10 +254,10 @@
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
+    CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
-        CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),

From pypy.commits at gmail.com  Wed Aug 24 06:12:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:12:47 -0700 (PDT)
Subject: [pypy-commit] pypy use-mmap: Superceded by use-madv-free
Message-ID: <57bd731f.02c41c0a.78b42.0dc2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: use-mmap
Changeset: r86477:62431e382e83
Date: 2016-08-24 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/62431e382e83/

Log:	Superceded by use-madv-free


From pypy.commits at gmail.com  Wed Aug 24 06:16:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:08 -0700 (PDT)
Subject: [pypy-commit] pypy issue2335: close merged branch
Message-ID: <57bd73e8.c62f1c0a.9f43d.cb86@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: issue2335
Changeset: r86478:68b03c738401
Date: 2016-08-24 12:13 +0200
http://bitbucket.org/pypy/pypy/changeset/68b03c738401/

Log:	close merged branch


From pypy.commits at gmail.com  Wed Aug 24 06:16:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:12 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head ca83cb5287e7
 on branch fix-gen-dfa
Message-ID: <57bd73ec.a710c20a.54e5.a09e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86480:c615391e9bc0
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/c615391e9bc0/

Log:	Merge closed head ca83cb5287e7 on branch fix-gen-dfa


From pypy.commits at gmail.com  Wed Aug 24 06:16:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:14 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head c327a2fa5f02
 on branch new-jit-log
Message-ID: <57bd73ee.03121c0a.2e400.d184@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86481:ed4389f7bf97
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/ed4389f7bf97/

Log:	Merge closed head c327a2fa5f02 on branch new-jit-log


From pypy.commits at gmail.com  Wed Aug 24 06:16:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:10 -0700 (PDT)
Subject: [pypy-commit] pypy save_socket_errno: close merged branch
Message-ID: <57bd73ea.6211c20a.7af0d.9bbd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: save_socket_errno
Changeset: r86479:bf922c1461c7
Date: 2016-08-24 12:14 +0200
http://bitbucket.org/pypy/pypy/changeset/bf922c1461c7/

Log:	close merged branch


From pypy.commits at gmail.com  Wed Aug 24 06:16:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:20 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 9ac7d2defab0
 on branch memoryview-attributes
Message-ID: <57bd73f4.d42f1c0a.d0cb9.cf49@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86485:d129717390db
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/d129717390db/

Log:	Merge closed head 9ac7d2defab0 on branch memoryview-attributes


From pypy.commits at gmail.com  Wed Aug 24 06:16:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:15 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 2569ef7fa3cc
 on branch py3.5-async-translate
Message-ID: <57bd73ef.411d1c0a.786fd.cf80@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86482:295f6f8fd710
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/295f6f8fd710/

Log:	Merge closed head 2569ef7fa3cc on branch py3.5-async-translate


From pypy.commits at gmail.com  Wed Aug 24 06:16:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:17 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 9f7d18f5d82f
 on branch refactor_rmmap
Message-ID: <57bd73f1.c3f0c20a.cb08.9fd6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86483:17571d1796fc
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/17571d1796fc/

Log:	Merge closed head 9f7d18f5d82f on branch refactor_rmmap


From pypy.commits at gmail.com  Wed Aug 24 06:16:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:23 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 68b03c738401
 on branch issue2335
Message-ID: <57bd73f7.eeb8c20a.90862.9547@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86487:490d2b04d53c
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/490d2b04d53c/

Log:	Merge closed head 68b03c738401 on branch issue2335


From pypy.commits at gmail.com  Wed Aug 24 06:16:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:18 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head f07260687448
 on branch rpython-deque
Message-ID: <57bd73f2.6974c20a.8a0d2.9776@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86484:31648ede1a09
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/31648ede1a09/

Log:	Merge closed head f07260687448 on branch rpython-deque


From pypy.commits at gmail.com  Wed Aug 24 06:16:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:25 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head bf922c1461c7
 on branch save_socket_errno
Message-ID: <57bd73f9.c62f1c0a.9f43d.cbab@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86488:42285066ce04
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/42285066ce04/

Log:	Merge closed head bf922c1461c7 on branch save_socket_errno


From pypy.commits at gmail.com  Wed Aug 24 06:16:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:22 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 62431e382e83
 on branch use-mmap
Message-ID: <57bd73f6.0205c20a.f2090.9d2b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86486:b4b42e343205
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/b4b42e343205/

Log:	Merge closed head 62431e382e83 on branch use-mmap


From pypy.commits at gmail.com  Wed Aug 24 06:16:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:16:27 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: re-close this branch
Message-ID: <57bd73fb.915c1c0a.ef9e5.d620@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r86489:2dbecd2a45b0
Date: 2016-08-24 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/2dbecd2a45b0/

Log:	re-close this branch


From pypy.commits at gmail.com  Wed Aug 24 06:21:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 03:21:40 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Added issue #2375 here, for py3.5.
Message-ID: <57bd7534.0117c20a.c7a53.a5e7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5682:7dd8cae2a466
Date: 2016-08-24 12:21 +0200
http://bitbucket.org/pypy/extradoc/changeset/7dd8cae2a466/

Log:	Added issue #2375 here, for py3.5.

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -34,6 +34,8 @@
   adding gi_yieldfrom/cr_await to generator/coroutines.  (Waiting
   because some work might be going on with raffael_t.)
 
+* compare ``dir(posix)`` on py3.5 and cpython 3.5.
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------

From pypy.commits at gmail.com  Wed Aug 24 06:22:15 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 03:22:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: missing import
Message-ID: <57bd7557.10a81c0a.52267.d327@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86490:608a0ce136ed
Date: 2016-08-24 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/608a0ce136ed/

Log:	missing import

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -20,6 +20,7 @@
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import widen
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
 from pypy.module.sys.version import CPYTHON_VERSION

From pypy.commits at gmail.com  Wed Aug 24 06:42:58 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 24 Aug 2016 03:42:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: (cfbolz,
 arigo): fix the mapdict cache for subclasses of builtin types that
Message-ID: <57bd7a32.271ac20a.51b3b.9eb5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86491:46e88ff9f92f
Date: 2016-08-24 11:42 +0100
http://bitbucket.org/pypy/pypy/changeset/46e88ff9f92f/

Log:	(cfbolz, arigo): fix the mapdict cache for subclasses of builtin
	types that provide a dict

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -1014,6 +1014,12 @@
 
 def LOOKUP_METHOD_mapdict_fill_cache_method(space, pycode, name, nameindex,
                                             w_obj, w_type, w_method):
+    # if the layout has a dict itself, then mapdict is not used for normal
+    # attributes. Then the cache won't be able to spot changes to the dict.
+    # Thus we don't cache. see test_bug_builtin_types_callmethod
+    if not w_type.layout.typedef.hasdict:
+        return
+
     if w_method is None or isinstance(w_method, MutableCell):
         # don't cache the MutableCell XXX could be fixed
         return
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1217,6 +1217,23 @@
         got = x.a
         assert got == 'd'
 
+    def test_bug_builtin_types_callmethod(self):
+        import sys
+        class D(type(sys)):
+            def mymethod(self):
+                return "mymethod"
+
+        def foobar():
+            return "foobar"
+
+        d = D('d')
+        res1 = d.mymethod()
+        d.mymethod = foobar
+        res2 = d.mymethod()
+        assert res1 == "mymethod"
+        assert res2 == "foobar"
+
+
 class AppTestGlobalCaching(AppTestWithMapDict):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
 

From pypy.commits at gmail.com  Wed Aug 24 06:53:03 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 24 Aug 2016 03:53:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: add the equivalent test for LOAD_ATTR.
 that already works, update the comment
Message-ID: <57bd7c8f.271ac20a.51b3b.a2a1@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86492:c25842df0d7d
Date: 2016-08-24 11:51 +0100
http://bitbucket.org/pypy/pypy/changeset/c25842df0d7d/

Log:	add the equivalent test for LOAD_ATTR. that already works, update
	the comment explaining why

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -992,7 +992,8 @@
             if index != INVALID:
                 attr = map.find_map_attr(attrname, index)
                 if attr is not None:
-                    # Note that if map.terminator is a DevolvedDictTerminator,
+                    # Note that if map.terminator is a DevolvedDictTerminator
+                    # or the class provides its own dict, not using mapdict, then:
                     # map.find_map_attr will always return None if index==DICT.
                     _fill_cache(pycode, nameindex, map, version_tag, attr.storageindex)
                     return w_obj._mapdict_read_storage(attr.storageindex)
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1233,6 +1233,25 @@
         assert res1 == "mymethod"
         assert res2 == "foobar"
 
+    def test_bug_builtin_types_load_attr(self):
+        import sys
+        class D(type(sys)):
+            def mymethod(self):
+                return "mymethod"
+
+        def foobar():
+            return "foobar"
+
+        d = D('d')
+        m = d.mymethod
+        res1 = m()
+        d.mymethod = foobar
+        m = d.mymethod
+        res2 = m()
+        assert res1 == "mymethod"
+        assert res2 == "foobar"
+
+
 
 class AppTestGlobalCaching(AppTestWithMapDict):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}

From pypy.commits at gmail.com  Wed Aug 24 08:33:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 05:33:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix the result type of
 _winreg.QueryValueEx() for REG_SZ or
Message-ID: <57bd9412.44ce1c0a.9a4e7.0a1a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86493:eaf0960e857d
Date: 2016-08-24 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/eaf0960e857d/

Log:	Fix the result type of _winreg.QueryValueEx() for REG_SZ or
	REG_EXPAND_SZ keys

diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -358,9 +358,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -460,7 +466,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -612,7 +618,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -151,6 +151,7 @@
 
     def test_readValues(self):
         from _winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from _winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -164,7 +165,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)

From pypy.commits at gmail.com  Wed Aug 24 09:13:37 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 24 Aug 2016 06:13:37 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-subclass: add failing test for subclass
 with tp_as_number methods
Message-ID: <57bd9d81.448e1c0a.4587.0af9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-subclass
Changeset: r86496:bbc135100879
Date: 2016-08-24 23:10 +1000
http://bitbucket.org/pypy/pypy/changeset/bbc135100879/

Log:	add failing test for subclass with tp_as_number methods

diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -87,4 +87,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         

From pypy.commits at gmail.com  Wed Aug 24 09:31:18 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 24 Aug 2016 06:31:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: start release notes
Message-ID: <57bda1a6.05371c0a.afb9f.1604@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86497:0b1493214116
Date: 2016-08-24 23:30 +1000
http://bitbucket.org/pypy/pypy/changeset/0b1493214116/

Log:	start release notes

diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,74 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+and fixed many issues and bugs raised by the growing community of PyPy
+users.
+
+XXXXX MORE ???
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+* Bug Fixes
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+* Performance improvements:
+
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers

From pypy.commits at gmail.com  Wed Aug 24 10:36:08 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 07:36:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: missing import
Message-ID: <57bdb0d8.465d1c0a.3dddb.3acb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86498:d711ffd606f3
Date: 2016-08-24 16:35 +0200
http://bitbucket.org/pypy/pypy/changeset/d711ffd606f3/

Log:	missing import

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -3,7 +3,7 @@
 from rpython.rlib import buffer
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer)
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, w_obj):

From pypy.commits at gmail.com  Wed Aug 24 10:40:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 07:40:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: (arigo,
 cfbolz): logic was inverted (bad cfbolz, no cookie)
Message-ID: <57bdb1ca.d4e01c0a.a8cdb.3464@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86499:2cd8ca921907
Date: 2016-08-24 14:43 +0100
http://bitbucket.org/pypy/pypy/changeset/2cd8ca921907/

Log:	(arigo, cfbolz): logic was inverted (bad cfbolz, no cookie)

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -1018,7 +1018,7 @@
     # if the layout has a dict itself, then mapdict is not used for normal
     # attributes. Then the cache won't be able to spot changes to the dict.
     # Thus we don't cache. see test_bug_builtin_types_callmethod
-    if not w_type.layout.typedef.hasdict:
+    if w_type.layout.typedef.hasdict:
         return
 
     if w_method is None or isinstance(w_method, MutableCell):

From pypy.commits at gmail.com  Wed Aug 24 10:40:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 07:40:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: (cfbolz, arigo)
Message-ID: <57bdb1cc.c41f1c0a.12d23.3a4b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86500:123020cd1730
Date: 2016-08-24 15:47 +0100
http://bitbucket.org/pypy/pypy/changeset/123020cd1730/

Log:	(cfbolz, arigo)

	Add a hypothesis-based test for creating and changing attributes on
	a class or on an instance (regular attributes, slots, methods, etc.)

diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -0,0 +1,131 @@
+import pytest
+import sys
+from hypothesis import given, strategies, settings
+from pypy.tool.pytest.objspace import gettestobjspace
+
+base_initargs = strategies.sampled_from([
+    ("object", (), False),
+    ("type(sys)", ("fake", ), True),
+    ("NewBase", (), True),
+    ("OldBase", (), False),
+    ("object, OldBase", (), False),
+    ("type(sys), OldBase", ("fake", ), True),
+    ])
+
+attrnames = strategies.sampled_from(["a", "b", "c"])
+
+ at strategies.composite
+def make_code(draw):
+    # now here we can do this kind of thing:
+    baseclass, initargs, hasdict = draw(base_initargs)
+    # and with arbitrary strategies
+
+    def class_attr():
+        what = draw(strategies.sampled_from(["value", "method", "property"]))
+        if what == "value":
+            val = draw(strategies.integers())
+            return val, str(val)
+        if what == "method":
+            val = draw(strategies.integers())
+            return (lambda self, val=val: val,
+                    "lambda self: %d" % val)
+        if what == "property":
+            val = draw(strategies.integers())
+            return (property(lambda self, val=val: val,
+                             lambda self, val: None,
+                             lambda self: None),
+                    "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+    code = ["import sys", "class OldBase:pass", "class NewBase(object):pass", "class A(%s):" % baseclass]
+    dct = {}
+    if draw(strategies.booleans()):
+        slots = draw(strategies.lists(attrnames))
+        if not hasdict and draw(strategies.booleans()):
+            slots.append("__dict__")
+        dct["__slots__"] = slots
+        code.append("    __slots__ = %s" % (slots, ))
+    for name in ["a", "b", "c"]:
+        if not draw(strategies.booleans()):
+            continue
+        dct[name], codeval = class_attr()
+        code.append("    %s = %s" % (name, codeval))
+    class OldBase: pass
+    class NewBase(object): pass
+    evaldct = {'OldBase': OldBase, 'NewBase': NewBase}
+    if baseclass == 'OldBase':
+        metaclass = type(OldBase)
+    else:
+        metaclass = type
+    cls = metaclass("A", eval(baseclass+',', globals(), evaldct), dct)
+    inst = cls(*initargs)
+    code.append("    pass")
+    code.append("a = A(*%s)" % (initargs, ))
+    for attr in draw(strategies.lists(attrnames, min_size=1)):
+        op = draw(strategies.sampled_from(["read", "read", "read",
+                      "write", "writemeth", "writeclass", "writebase",
+                      "del", "delclass"]))
+        if op == "read":
+            try:
+                res = getattr(inst, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s')" % (attr, ))
+            else:
+                if callable(res):
+                    code.append("assert a.%s() == %s" % (attr, res()))
+                else:
+                    code.append("assert a.%s == %s" % (attr, res))
+        elif op == "write":
+            val = draw(strategies.integers())
+            try:
+                setattr(inst, attr, val)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s=%s')" % (attr, val))
+            else:
+                code.append("a.%s = %s" % (attr, val))
+        elif op == "writemeth":
+            val = draw(strategies.integers())
+            try:
+                setattr(inst, attr, lambda val=val: val)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s=0')" % (attr, ))
+            else:
+                code.append("a.%s = lambda : %s" % (attr, val))
+        elif op == "writeclass":
+            val, codeval = class_attr()
+            setattr(cls, attr, val)
+            code.append("A.%s = %s" % (attr, codeval))
+        elif op == "writebase":
+            val, codeval = class_attr()
+            setattr(OldBase, attr, val)
+            setattr(NewBase, attr, val)
+            code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))
+        elif op == "del":
+            try:
+                delattr(inst, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'del a.%s')" % (attr, ))
+            else:
+                code.append("del a.%s" % (attr, ))
+        elif op == "delclass":
+            try:
+                delattr(cls, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'del A.%s')" % (attr, ))
+            else:
+                code.append("del A.%s" % (attr, ))
+    return "\n    ".join(code)
+
+
+ at given(make_code())
+#@settings(max_examples=5000)
+def test_random_attrs(code):
+    try:
+        import __pypy__
+    except ImportError:
+        pass
+    else:
+        pytest.skip("makes no sense under pypy!")
+    space = gettestobjspace()
+    print code
+    exec "if 1:\n    " + code
+    space.appexec([], "():\n    " + code)

From pypy.commits at gmail.com  Wed Aug 24 10:40:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 07:40:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <57bdb1cd.e129c20a.4ccf.0a37@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86501:145e67b3b78a
Date: 2016-08-24 15:48 +0100
http://bitbucket.org/pypy/pypy/changeset/145e67b3b78a/

Log:	merge heads

diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,74 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+and fixed many issues and bugs raised by the growing community of PyPy
+users.
+
+XXXXX MORE ???
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+* Bug Fixes
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+* Performance improvements:
+
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers

From pypy.commits at gmail.com  Wed Aug 24 10:47:26 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 24 Aug 2016 07:47:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: document commits to default up to July
 15
Message-ID: <57bdb37e.c2a5c20a.46c00.05c3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86502:2fe6d5f48eb5
Date: 2016-08-25 00:45 +1000
http://bitbucket.org/pypy/pypy/changeset/2fe6d5f48eb5/

Log:	document commits to default up to July 15

diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -59,14 +59,82 @@
 
 * New features:
 
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
 * Bug Fixes
 
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix OpenBSD build breakage and support OpenBSD in VMProf.
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
   * Issues reported with our previous release were resolved_ after
     reports from users on our issue tracker at
     https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
 
 * Performance improvements:
 
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
 .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
 
 Please update, and continue to help us make PyPy better.

From pypy.commits at gmail.com  Wed Aug 24 11:19:11 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 08:19:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: remove double definition of cpyext C symbol
Message-ID: <57bdbaef.151a1c0a.bc2cc.4905@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86503:31eb26fe8cfa
Date: 2016-08-24 17:17 +0200
http://bitbucket.org/pypy/pypy/changeset/31eb26fe8cfa/

Log:	remove double definition of cpyext C symbol

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -516,7 +516,7 @@
     'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs',
     '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT',
 
-    'PyObject_GetBuffer', 'PyBuffer_Release',
+    'PyBuffer_Release',
 
     'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr',
     'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr',

From pypy.commits at gmail.com  Wed Aug 24 11:34:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 08:34:38 -0700 (PDT)
Subject: [pypy-commit] cffi default: update whatsnew
Message-ID: <57bdbe8e.c997c20a.2d8c4.1603@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2747:8ee16fdadf92
Date: 2016-08-24 17:34 +0200
http://bitbucket.org/cffi/cffi/changeset/8ee16fdadf92/

Log:	update whatsnew

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -529,9 +529,7 @@
 the same version of CPython x.y).  However, the standard ``distutils``
 package will still produce a file called e.g.
 ``NAME.cpython-35m-x86_64-linux-gnu.so``.  You can manually rename it to
-``NAME.abi3.so``.  There are certainly other ways to compile the C code
-that produce directly the correct file name, but I know of no
-widely-used solution.
+``NAME.abi3.so``, or use setuptools version 26 or later.
 
 **ffibuilder.compile(tmpdir='.', verbose=False):**
 explicitly generate the .py or .c file,
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -6,6 +6,13 @@
 v1.8
 ====
 
+* CPython 3.x: experimental: the generated C extension modules now use
+  the "limited API", which means that, as a compiled .so/.dll, it should
+  work directly on any version of CPython >= 3.2.  The name produced by
+  distutils is still version-specific.  To get the version-independent
+  name, you can rename it manually to ``NAME.abi3.so``, or use the very
+  recent setuptools 26.
+
 * Removed the restriction that ``ffi.from_buffer()`` cannot be used on
   byte strings.  Now you can get a ``char *`` out of a byte string,
   which is valid as long as the string object is kept alive.  (But

From pypy.commits at gmail.com  Wed Aug 24 11:55:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 08:55:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: Improve the test: if x == -1 then
 int(-0.8) == 0, and the second part of
Message-ID: <57bdc35d.4219c20a.987dd.1ed0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86504:189373c314f3
Date: 2016-08-24 17:54 +0200
http://bitbucket.org/pypy/pypy/changeset/189373c314f3/

Log:	Improve the test: if x == -1 then int(-0.8) == 0, and the second
	part of the test would not be run at all

diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -50,7 +50,7 @@
     resource.setrlimit(resource.RLIMIT_CPU, (x, y))
     # sometimes, x and y are very large (more than 53 bits).
     # for these huge values, int(float(x)) > x...
-    xf = x + 0.2
-    yf = y + 0.3
+    xf = x + (0.2 if x >= 0 else -0.2)
+    yf = y + (0.3 if y >= 0 else -0.3)
     if int(xf) == x and int(yf) == y:
         resource.setrlimit(resource.RLIMIT_CPU, (x, y))  # truncated to ints

From pypy.commits at gmail.com  Wed Aug 24 13:06:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 10:06:40 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: hg merge default
Message-ID: <57bdd420.4fd71c0a.3deb5.6fde@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86505:670d925b3853
Date: 2016-08-24 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/670d925b3853/

Log:	hg merge default

diff too long, truncating to 2000 out of 15666 lines

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -85,10 +85,11 @@
     pass
 
 def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')):
+    import ast
     result = []
     for word in re_word.findall(line):
         if word.startswith('"'):
-            word = eval(word)
+            word = ast.literal_eval(word)
         result.append(word)
     return result
 
diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib-python/2.7/test/test_hash.py b/lib-python/2.7/test/test_hash.py
--- a/lib-python/2.7/test/test_hash.py
+++ b/lib-python/2.7/test/test_hash.py
@@ -174,7 +174,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
         else:
             return self.value
 
-    def __buffer__(self):
+    def __buffer__(self, flags):
         return buffer(self._buffer)
 
     def _get_b_base(self):
@@ -199,10 +199,13 @@
     return tp._alignmentofinstances()
 
 @builtinify
-def byref(cdata):
+def byref(cdata, offset=0):
     # "pointer" is imported at the end of this module to avoid circular
     # imports
-    return pointer(cdata)
+    ptr = pointer(cdata)
+    if offset != 0:
+        ptr._buffer[0] += offset
+    return ptr
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -0,0 +1,91 @@
+# Note: uses the CFFI out-of-line ABI mode.  We can't use the API
+# mode because ffi.compile() needs to run the compiler, which
+# needs 'subprocess', which needs 'msvcrt' and '_subprocess',
+# which depend on '_pypy_winbase_cffi' already.
+#
+# Note that if you need to regenerate _pypy_winbase_cffi and
+# can't use a preexisting PyPy to do that, then running this
+# file should work as long as 'subprocess' is not imported
+# by cffi.  I had to hack in 'cffi._pycparser' to move an
+#'import subprocess' to the inside of a function.  (Also,
+# CPython+CFFI should work as well.)
+#
+# This module supports both msvcrt.py and _subprocess.py.
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pypy_winbase_cffi", None)
+
+# ---------- MSVCRT ----------
+
+ffi.cdef("""
+typedef unsigned short wint_t;
+
+int _open_osfhandle(intptr_t osfhandle, int flags);
+intptr_t _get_osfhandle(int fd);
+int _setmode(int fd, int mode);
+int _locking(int fd, int mode, long nbytes);
+
+int _kbhit(void);
+int _getch(void);
+wint_t _getwch(void);
+int _getche(void);
+wint_t _getwche(void);
+int _putch(int);
+wint_t _putwch(wchar_t);
+int _ungetch(int);
+wint_t _ungetwch(wint_t);
+""")
+
+# ---------- SUBPROCESS ----------
+
+ffi.cdef("""
+typedef struct {
+    DWORD  cb;
+    char * lpReserved;
+    char * lpDesktop;
+    char * lpTitle;
+    DWORD  dwX;
+    DWORD  dwY;
+    DWORD  dwXSize;
+    DWORD  dwYSize;
+    DWORD  dwXCountChars;
+    DWORD  dwYCountChars;
+    DWORD  dwFillAttribute;
+    DWORD  dwFlags;
+    WORD   wShowWindow;
+    WORD   cbReserved2;
+    LPBYTE lpReserved2;
+    HANDLE hStdInput;
+    HANDLE hStdOutput;
+    HANDLE hStdError;
+} STARTUPINFO, *LPSTARTUPINFO;
+
+typedef struct {
+    HANDLE hProcess;
+    HANDLE hThread;
+    DWORD  dwProcessId;
+    DWORD  dwThreadId;
+} PROCESS_INFORMATION, *LPPROCESS_INFORMATION;
+
+DWORD WINAPI GetVersion(void);
+BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD);
+BOOL WINAPI CloseHandle(HANDLE);
+HANDLE WINAPI GetCurrentProcess(void);
+BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
+                            DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
+BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
+BOOL WINAPI TerminateProcess(HANDLE, UINT);
+HANDLE WINAPI GetStdHandle(DWORD);
+""")
+
+# --------------------
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -0,0 +1,10 @@
+# auto-generated file
+import _cffi_backend
+
+ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
+    _version = 0x2601,
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+)
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -10,148 +10,99 @@
 
 # Declare external Win32 functions
 
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
+from _pypy_winbase_cffi import ffi as _ffi
+_kernel32 = _ffi.dlopen('kernel32')
 
 GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
 
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
-
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
-
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
-
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
-
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessA
-_CreateProcess.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
 
 # Now the _subprocess module implementation
 
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
+def _WinError():
+    code, message = _ffi.getwinerror()
+    raise WindowsError(code, message)
 
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
+_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+
+class _handle(object):
+    def __init__(self, c_handle):
+        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
+        self.c_handle = c_handle
+        if int(self) != -1:
+            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
 
     def __int__(self):
-        return self.handle
+        return int(_ffi.cast("intptr_t", self.c_handle))
 
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
+    def __repr__(self):
+        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
 
     def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
+        h = int(self)
+        if h != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+        return h
 
     def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
+        if int(self) != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+            _kernel32.CloseHandle(c_handle)
 
 def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
+    handles = _ffi.new("HANDLE[2]")
 
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
+    res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size)
 
     if not res:
         raise _WinError()
 
-    return _handle(read.value), _handle(write.value)
+    return _handle(handles[0]), _handle(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
+    return _handle(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
+    # CPython: the first three arguments are expected to be integers
+    target = _ffi.new("HANDLE[1]")
 
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
+    res = _kernel32.DuplicateHandle(
+        _ffi.cast("HANDLE", source_process),
+        _ffi.cast("HANDLE", source),
+        _ffi.cast("HANDLE", target_process),
+        target, access, inherit, options)
 
     if not res:
         raise _WinError()
 
-    return _handle(target.value)
+    return _handle(target[0])
+
+def _z(input):
+    if input is None:
+        return _ffi.NULL
+    if isinstance(input, basestring):
+        return str(input)
+    raise TypeError("string/unicode/None expected, got %r" % (
+        type(input).__name__,))
 
 def CreateProcess(name, command_line, process_attr, thread_attr,
                   inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
+    si = _ffi.new("STARTUPINFO *")
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
+        # CPython: these three handles are expected to be _handle objects
         if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
+            si.hStdInput = startup_info.hStdInput.c_handle
         if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
+            si.hStdOutput = startup_info.hStdOutput.c_handle
         if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
+            si.hStdError = startup_info.hStdError.c_handle
 
-    pi = _PROCESS_INFORMATION()
+    pi = _ffi.new("PROCESS_INFORMATION *")
 
     if env is not None:
         envbuf = ""
@@ -159,47 +110,55 @@
             envbuf += "%s=%s\0" % (k, v)
         envbuf += '\0'
     else:
-        envbuf = None
+        envbuf = _ffi.NULL
 
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
+    res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL,
+                                   _ffi.NULL, inherit, flags, envbuf,
+                                   _z(start_dir), si, pi)
 
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
 
 def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
+    # CPython: the first argument is expected to be an integer.
+    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
+                                        milliseconds)
     if res < 0:
         raise _WinError()
 
     return res
 
 def GetExitCodeProcess(handle):
-    code = _c_int()
+    # CPython: the first argument is expected to be an integer.
+    code = _ffi.new("DWORD[1]")
 
-    res = _GetExitCodeProcess(int(handle), _byref(code))
+    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
 
     if not res:
         raise _WinError()
 
-    return code.value
+    return code[0]
 
 def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
+    # CPython: the first argument is expected to be an integer.
+    # The second argument is silently wrapped in a UINT.
+    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+                                     _ffi.cast("UINT", exitcode))
 
     if not res:
         raise _WinError()
 
 def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
+    stdhandle = _ffi.cast("DWORD", stdhandle)
+    res = _kernel32.GetStdHandle(stdhandle)
 
     if not res:
         return None
     else:
-        return res
+        # note: returns integer, not handle object
+        return int(_ffi.cast("intptr_t", res))
 
 STD_INPUT_HANDLE = -10
 STD_OUTPUT_HANDLE = -11
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py
--- a/lib_pypy/cffi/_pycparser/__init__.py
+++ b/lib_pypy/cffi/_pycparser/__init__.py
@@ -10,7 +10,6 @@
 __all__ = ['c_lexer', 'c_parser', 'c_ast']
 __version__ = '2.14'
 
-from subprocess import Popen, PIPE
 from .c_parser import CParser
 
 
@@ -28,6 +27,7 @@
         When successful, returns the preprocessed file's contents.
         Errors from cpp will be printed out.
     """
+    from subprocess import Popen, PIPE
     path_list = [cpp_path]
     if isinstance(cpp_args, list):
         path_list += cpp_args
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -513,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -570,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -7,26 +7,39 @@
 # XXX incomplete: implemented only functions needed by subprocess.py
 # PAC: 2010/08 added MS locking for Whoosh
 
-import ctypes
+# 07/2016: rewrote in CFFI
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The 'msvcrt' module is only available on Windows")
+
+import _rawffi
+from _pypy_winbase_cffi import ffi as _ffi
+_lib = _ffi.dlopen(_rawffi.get_libc().name)
+
 import errno
-from ctypes_support import standard_c_lib as _c
-from ctypes_support import get_errno
-
-try:
-    open_osfhandle = _c._open_osfhandle
-except AttributeError: # we are not on windows
-    raise ImportError
 
 try: from __pypy__ import builtinify, validate_fd
 except ImportError: builtinify = validate_fd = lambda f: f
 
 
-open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int]
-open_osfhandle.restype = ctypes.c_int
+def _ioerr():
+    e = _ffi.errno
+    raise IOError(e, errno.errorcode[e])
 
-_get_osfhandle = _c._get_osfhandle
-_get_osfhandle.argtypes = [ctypes.c_int]
-_get_osfhandle.restype = ctypes.c_int
+
+ at builtinify
+def open_osfhandle(fd, flags):
+    """"open_osfhandle(handle, flags) -> file descriptor
+
+    Create a C runtime file descriptor from the file handle handle. The
+    flags parameter should be a bitwise OR of os.O_APPEND, os.O_RDONLY,
+    and os.O_TEXT. The returned file descriptor may be used as a parameter
+    to os.fdopen() to create a file object."""
+    fd = _lib._open_osfhandle(fd, flags)
+    if fd == -1:
+        _ioerr()
+    return fd
 
 @builtinify
 def get_osfhandle(fd):
@@ -38,62 +51,74 @@
         validate_fd(fd)
     except OSError as e:
         raise IOError(*e.args)
-    return _get_osfhandle(fd)
+    result = _lib._get_osfhandle(fd)
+    if result == -1:
+        _ioerr()
+    return result
 
-setmode = _c._setmode
-setmode.argtypes = [ctypes.c_int, ctypes.c_int]
-setmode.restype = ctypes.c_int
+ at builtinify
+def setmode(fd, flags):
+    """setmode(fd, mode) -> Previous mode
+
+    Set the line-end translation mode for the file descriptor fd. To set
+    it to text mode, flags should be os.O_TEXT; for binary, it should be
+    os.O_BINARY."""
+    flags = _lib._setmode(fd, flags)
+    if flags == -1:
+        _ioerr()
+    return flags
 
 LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5)
 
-_locking = _c._locking
-_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_locking.restype = ctypes.c_int
-
 @builtinify
 def locking(fd, mode, nbytes):
-    '''lock or unlock a number of bytes in a file.'''
-    rv = _locking(fd, mode, nbytes)
+    """"locking(fd, mode, nbytes) -> None
+
+    Lock part of a file based on file descriptor fd from the C runtime.
+    Raises IOError on failure. The locked region of the file extends from
+    the current file position for nbytes bytes, and may continue beyond
+    the end of the file. mode must be one of the LK_* constants listed
+    below. Multiple regions in a file may be locked at the same time, but
+    may not overlap. Adjacent regions are not merged; they must be unlocked
+    individually."""
+    rv = _lib._locking(fd, mode, nbytes)
     if rv != 0:
-        e = get_errno()
-        raise IOError(e, errno.errorcode[e])
+        _ioerr()
 
 # Console I/O routines
 
-kbhit = _c._kbhit
-kbhit.argtypes = []
-kbhit.restype = ctypes.c_int
+kbhit = _lib._kbhit
 
-getch = _c._getch
-getch.argtypes = []
-getch.restype = ctypes.c_char
+ at builtinify
+def getch():
+    return chr(_lib._getch())
 
-getwch = _c._getwch
-getwch.argtypes = []
-getwch.restype = ctypes.c_wchar
+ at builtinify
+def getwch():
+    return unichr(_lib._getwch())
 
-getche = _c._getche
-getche.argtypes = []
-getche.restype = ctypes.c_char
+ at builtinify
+def getche():
+    return chr(_lib._getche())
 
-getwche = _c._getwche
-getwche.argtypes = []
-getwche.restype = ctypes.c_wchar
+ at builtinify
+def getwche():
+    return unichr(_lib._getwche())
 
-putch = _c._putch
-putch.argtypes = [ctypes.c_char]
-putch.restype = None
+ at builtinify
+def putch(ch):
+    _lib._putch(ord(ch))
 
-putwch = _c._putwch
-putwch.argtypes = [ctypes.c_wchar]
-putwch.restype = None
+ at builtinify
+def putwch(ch):
+    _lib._putwch(ord(ch))
 
-ungetch = _c._ungetch
-ungetch.argtypes = [ctypes.c_char]
-ungetch.restype = None
+ at builtinify
+def ungetch(ch):
+    if _lib._ungetch(ord(ch)) == -1:   # EOF
+        _ioerr()
 
-ungetwch = _c._ungetwch
-ungetwch.argtypes = [ctypes.c_wchar]
-ungetwch.restype = None
-
-del ctypes
+ at builtinify
+def ungetwch(ch):
+    if _lib._ungetwch(ord(ch)) == -1:   # EOF
+        _ioerr()
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -36,7 +36,7 @@
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
-    "_csv", "cppyy", "_pypyjson",
+    "_csv", "cppyy", "_pypyjson", "_jitlog"
 ])
 
 from rpython.jit.backend import detect_cpu
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -104,27 +104,24 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev liblzma-dev
-
-For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+    tk-dev libgc-dev \
+    liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel
-
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+    gdbm-devel \
+    xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses
+    libexpat-devel libffi-devel python-curses \
+    xz-devel  # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
-
 On Mac OS X, most of these build-time dependencies are installed alongside
 the Developer Tools. However, note that in order for the installation to
 find them you may need to run::
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -335,3 +335,65 @@
 
 This will disable SELinux's protection and allow PyPy to configure correctly.
 Be sure to enable it again if you need it!
+
+
+How should I report a bug?
+--------------------------
+
+Our bug tracker is here: https://bitbucket.org/pypy/pypy/issues/
+
+Missing features or incompatibilities with CPython are considered
+bugs, and they are welcome.  (See also our list of `known
+incompatibilities`__.)
+
+.. __: http://pypy.org/compat.html
+
+For bugs of the kind "I'm getting a PyPy crash or a strange
+exception", please note that: **We can't do anything without
+reproducing the bug ourselves**.  We cannot do anything with
+tracebacks from gdb, or core dumps.  This is not only because the
+standard PyPy is compiled without debug symbols.  The real reason is
+that a C-level traceback is usually of no help at all in PyPy.
+Debugging PyPy can be annoying.
+
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
+In more details:
+
+* First, please give the exact PyPy version, and the OS.
+
+* It might help focus our search if we know if the bug can be
+  reproduced on a "``pypy --jit off``" or not.  If "``pypy --jit
+  off``" always works, then the problem might be in the JIT.
+  Otherwise, we know we can ignore that part.
+
+* If you got the bug using only Open Source components, please give a
+  step-by-step guide that we can follow to reproduce the problem
+  ourselves.  Don't assume we know anything about any program other
+  than PyPy.  We would like a guide that we can follow point by point
+  (without guessing or having to figure things out)
+  on a machine similar to yours, starting from a bare PyPy, until we
+  see the same problem.  (If you can, you can try to reduce the number
+  of steps and the time it needs to run, but that is not mandatory.)
+
+* If the bug involves Closed Source components, or just too many Open
+  Source components to install them all ourselves, then maybe you can
+  give us some temporary ssh access to a machine where the bug can be
+  reproduced.  Or, maybe we can download a VirtualBox or VMWare
+  virtual machine where the problem occurs.
+
+* If giving us access would require us to use tools other than ssh,
+  make appointments, or sign a NDA, then we can consider a commerical
+  support contract for a small sum of money.
+
+* If even that is not possible for you, then sorry, we can't help.
+
+Of course, you can try to debug the problem yourself, and we can help
+you get started if you ask on the #pypy IRC channel, but be prepared:
+debugging an annoying PyPy problem usually involves quite a lot of gdb
+in auto-generated C code, and at least some knowledge about the
+various components involved, from PyPy's own RPython source code to
+the GC and possibly the JIT.
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -39,17 +39,16 @@
 library.
 
 If you want to install 3rd party libraries, the most convenient way is
-to install pip_ (unless you want to install virtualenv as explained
-below; then you can directly use pip inside virtualenvs):
+to install pip_ using ensurepip_ (unless you want to install virtualenv as 
+explained below; then you can directly use pip inside virtualenvs):
 
 .. code-block:: console
 
-    $ curl -O https://bootstrap.pypa.io/get-pip.py
-    $ ./pypy-2.1/bin/pypy get-pip.py
-    $ ./pypy-2.1/bin/pip install pygments  # for example
+    $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install pygments  # for example
 
-Third party libraries will be installed in ``pypy-2.1/site-packages``, and
-the scripts in ``pypy-2.1/bin``.
+Third party libraries will be installed in ``pypy-xxx/site-packages``, and
+the scripts in ``pypy-xxx/bin``.
 
 
 Installing using virtualenv
@@ -61,7 +60,7 @@
 checkout::
 
 	# from a tarball
-	$ virtualenv -p /opt/pypy-c-jit-41718-3fb486695f20-linux/bin/pypy my-pypy-env
+	$ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env
 
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
@@ -69,7 +68,7 @@
 Note that bin/python is now a symlink to bin/pypy.
 
 .. _pip: http://pypi.python.org/pypi/pip
-
+.. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html
 
 Building PyPy yourself
 ~~~~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,142 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+and fixed many issues and bugs raised by the growing community of PyPy
+users.
+
+XXXXX MORE ???
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix OpenBSD build breakage and support OpenBSD in VMProf.
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -53,3 +53,109 @@
 
 Refactor PyTupleObject to look like cpython's and allow subclassing 
 PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -24,11 +24,15 @@
 -V     : print the Python version number and exit (also --version)
 -W arg : warning control; arg is action:message:category:module:lineno
          also PYTHONWARNINGS=arg
+-X arg : set implementation-specific option
 file   : program read from script file
 -      : program read from stdin (default; interactive mode if a tty)
 arg ...: arguments passed to program in sys.argv[1:]
+
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
+-X track-resources : track the creation of files and sockets and display
+                     a warning if they are not closed explicitly
 """
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
@@ -235,6 +239,14 @@
         import pypyjit
         pypyjit.set_param(jitparam)
 
+def set_runtime_options(options, Xparam, *args):
+    if Xparam == 'track-resources':
+        sys.pypy_set_track_resources(True)
+    else:
+        print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),)
+        print >> sys.stderr, '[options] can be: track-resources'
+        raise SystemExit
+
 class CommandLineError(Exception):
     pass
 
@@ -410,6 +422,7 @@
     '--info':    (print_info,      None),
     '--jit':     (set_jit_option,  Ellipsis),
     '-funroll-loops': (funroll_loops, None),
+    '-X':        (set_runtime_options, Ellipsis),
     '--':        (end_options,     None),
     }
 
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -387,7 +387,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -406,8 +407,11 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            assert depth >= 0
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -108,8 +108,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1166,3 +1166,22 @@
             counts = self.count_instructions(source)
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
+
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -208,7 +208,8 @@
     def buffer_w(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self, 
+                                        space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
@@ -216,7 +217,8 @@
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
@@ -224,7 +226,8 @@
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
@@ -232,7 +235,8 @@
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
@@ -1729,6 +1733,23 @@
                 "Python int too large for C unsigned short")
         return value
 
+    def c_uid_t_w(self, w_obj):
+        # xxx assumes that uid_t and gid_t are a C unsigned int.
+        # Equivalent to space.c_uint_w(), with the exception that
+        # it also accepts -1 and converts that to UINT_MAX, which
+        # is (uid_t)-1.  And values smaller than -1 raise
+        # OverflowError, not ValueError.
+        try:
+            return self.c_uint_w(w_obj)
+        except OperationError as e:
+            if e.match(self, self.w_ValueError):
+                # ValueError: cannot convert negative integer to unsigned
+                if self.int_w(w_obj) == -1:
+                    return UINT_MAX
+                raise oefmt(self.w_OverflowError,
+                            "user/group id smaller than minimum (-1)")
+            raise
+
     def truncatedint_w(self, w_obj, allow_conversion=True):
         # Like space.gateway_int_w(), but return the integer truncated
         # instead of raising OverflowError.  For obscure cases only.
@@ -1790,6 +1811,40 @@
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
         """)
 
+    def resource_warning(self, w_msg, w_tb):
+        self.appexec([w_msg, w_tb],
+                     """(msg, tb):
+            import sys
+            print >> sys.stderr, msg
+            if tb:
+                print >> sys.stderr, "Created at (most recent call last):"
+                print >> sys.stderr, tb
+        """)
+
+    def format_traceback(self):
+        # we need to disable track_resources before calling the traceback
+        # module. Else, it tries to open more files to format the traceback,
+        # the file constructor will call space.format_traceback etc., in an
+        # inifite recursion
+        flag = self.sys.track_resources
+        self.sys.track_resources = False
+        try:
+            return self.appexec([],
+                         """():
+                import sys, traceback
+                # the "1" is because we don't want to show THIS code
+                # object in the traceback
+                try:
+                    f = sys._getframe(1)
+                except ValueError:
+                    # this happens if you call format_traceback at the very beginning
+                    # of startup, when there is no bottom code object
+                    return '<no stacktrace available>'
+                return "".join(traceback.format_stack(f))
+            """)
+        finally:
+            self.sys.track_resources = flag
+
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -167,6 +167,9 @@
     def visit_c_ushort(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_c_uid_t(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_truncatedint_w(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -294,6 +297,9 @@
     def visit_c_ushort(self, typ):
         self.run_args.append("space.c_ushort_w(%s)" % (self.scopenext(),))
 
+    def visit_c_uid_t(self, typ):
+        self.run_args.append("space.c_uid_t_w(%s)" % (self.scopenext(),))
+
     def visit_truncatedint_w(self, typ):
         self.run_args.append("space.truncatedint_w(%s)" % (self.scopenext(),))
 
@@ -440,6 +446,9 @@
     def visit_c_ushort(self, typ):
         self.unwrap.append("space.c_ushort_w(%s)" % (self.nextarg(),))
 
+    def visit_c_uid_t(self, typ):
+        self.unwrap.append("space.c_uid_t_w(%s)" % (self.nextarg(),))
+
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
@@ -587,7 +596,10 @@
 
         # First extract the signature from the (CPython-level) code object
         from pypy.interpreter import pycode
-        argnames, varargname, kwargname = pycode.cpython_code_signature(func.func_code)
+        sig = pycode.cpython_code_signature(func.func_code)
+        argnames = sig.argnames
+        varargname = sig.varargname
+        kwargname = sig.kwargname
         self._argnames = argnames
 
         if unwrap_spec is None:
@@ -611,7 +623,9 @@
         app_sig = SignatureBuilder(func)
 
         UnwrapSpec_Check(orig_sig).apply_over(unwrap_spec, app_sig)
-        self.sig = argnames, varargname, kwargname = app_sig.signature()
+        self.sig = app_sig.signature()
+        argnames = self.sig.argnames
+        varargname = self.sig.varargname
 
         self.minargs = len(argnames)
         if varargname:
@@ -942,7 +956,7 @@
                 defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
             UNDEFINED = object()
-            alldefs_w = [UNDEFINED] * len(self._code.sig[0])
+            alldefs_w = [UNDEFINED] * len(self._code.sig.argnames)
             if defs_w:
                 alldefs_w[-len(defs_w):] = defs_w
             code = self._code
@@ -959,7 +973,7 @@
                     assert isinstance(w_default, W_Root)
                     assert argname.startswith('w_')
                     argname = argname[2:]
-                    j = self._code.sig[0].index(argname)
+                    j = self._code.sig.argnames.index(argname)
                     assert alldefs_w[j] in (UNDEFINED, None)
                     alldefs_w[j] = w_default
             first_defined = 0
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -37,7 +37,7 @@
 
 # cpython_code_signature helper
 def cpython_code_signature(code):
-    "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
+    """Return a Signature instance."""
     argcount = code.co_argcount
     varnames = code.co_varnames
     assert argcount >= 0     # annotator hint
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -55,18 +55,3 @@
         if not isinstance(other, Signature):
             return NotImplemented
         return not self == other
-
-
-    # make it look tuply for its use in the annotator
-
-    def __len__(self):
-        return 3
-
-    def __getitem__(self, i):
-        if i == 0:
-            return self.argnames
-        if i == 1:
-            return self.varargname
-        if i == 2:
-            return self.kwargname
-        raise IndexError
\ No newline at end of file
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -220,6 +220,13 @@
         expected = {"no_user_site": True}
         self.check(['-c', 'pass'], {}, sys_argv=['-c'], run_command='pass', **expected)
 
+    def test_track_resources(self, monkeypatch):
+        myflag = [False]
+        def pypy_set_track_resources(flag):
+            myflag[0] = flag
+        monkeypatch.setattr(sys, 'pypy_set_track_resources', pypy_set_track_resources, raising=False)
+        self.check(['-X', 'track-resources'], {}, sys_argv=[''], run_stdin=True)
+        assert myflag[0] == True
 
 class TestInteraction:
     """
@@ -1074,4 +1081,3 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -46,13 +46,6 @@
         assert sig.find_argname("c") == 2
         assert sig.find_argname("d") == -1
 
-    def test_tuply(self):
-        sig = Signature(["a", "b", "c"], "d", "e")
-        x, y, z = sig
-        assert x == ["a", "b", "c"]
-        assert y == "d"
-        assert z == "e"
-
 class dummy_wrapped_dict(dict):
     def __nonzero__(self):
         raise NotImplementedError
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -427,3 +427,28 @@
         space.finish()
         # assert that we reach this point without getting interrupted
         # by the OperationError(NameError)
+
+    def test_format_traceback(self):
+        from pypy.tool.pytest.objspace import maketestobjspace
+        from pypy.interpreter.gateway import interp2app
+        #
+        def format_traceback(space):
+            return space.format_traceback()
+        #
+        space = maketestobjspace()
+        w_format_traceback = space.wrap(interp2app(format_traceback))
+        w_tb = space.appexec([w_format_traceback], """(format_traceback):
+            def foo():
+                return bar()
+            def bar():
+                return format_traceback()
+            return foo()
+        """)
+        tb = space.str_w(w_tb)
+        expected = '\n'.join([
+            '  File "?", line 6, in anonymous',  # this is the appexec code object
+            '  File "?", line 3, in foo',
+            '  File "?", line 5, in bar',
+            ''
+        ])
+        assert tb == expected
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -203,7 +203,8 @@
     name = func.__name__
     extra = ', '.join(extraargs)
     from pypy.interpreter import pycode
-    argnames, _, _ = pycode.cpython_code_signature(func.func_code)
+    sig = pycode.cpython_code_signature(func.func_code)
+    argnames = sig.argnames
     if use_closure:
         if argnames[1] == 'space':
             args = "closure, space, obj"
diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -46,9 +46,65 @@
             raise       # propagate other errors
         return space.type(w_obj)
 
+
+# ---------- isinstance ----------
+
+
+def p_recursive_isinstance_w(space, w_inst, w_cls):
+    # Copied straight from CPython 2.7.  Does not handle 'cls' being a tuple.
+    if (isinstance(w_cls, W_ClassObject) and
+        isinstance(w_inst, W_InstanceObject)):
+        return w_inst.w_class.is_subclass_of(w_cls)
+
+    if space.isinstance_w(w_cls, space.w_type):
+        return p_recursive_isinstance_type_w(space, w_inst, w_cls)
+
+    check_class(space, w_cls, "isinstance() arg 2 must be a class, type,"
+                              " or tuple of classes and types")
+    try:
+        w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
+    except OperationError as e:
+        if e.async(space):      # ignore most exceptions
+            raise
+        return False
+    else:
+        return p_abstract_issubclass_w(space, w_abstractclass, w_cls)
+
+
+def p_recursive_isinstance_type_w(space, w_inst, w_type):
+    # subfunctionality of p_recursive_isinstance_w(): assumes that w_type is
+    # a type object.  Copied straight from CPython 2.7.
+    if space.isinstance_w(w_inst, w_type):
+        return True
+    try:
+        w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
+    except OperationError as e:
+        if e.async(space):      # ignore most exceptions
+            raise
+    else:
+        if w_abstractclass is not space.type(w_inst):
+            if space.isinstance_w(w_abstractclass, space.w_type):
+                return space.issubtype_w(w_abstractclass, w_type)
+    return False
+
+
 @jit.unroll_safe
 def abstract_isinstance_w(space, w_obj, w_klass_or_tuple, allow_override=False):
     """Implementation for the full 'isinstance(obj, klass_or_tuple)'."""
+    # Copied from CPython 2.7's PyObject_Isinstance().  Additionally,
+    # if 'allow_override' is False (the default), then don't try to
+    # use a custom __instancecheck__ method.
+
+    # WARNING: backward compatibility function name here.  CPython
+    # uses the name "abstract" to refer to the logic of handling
+    # class-like objects, with a "__bases__" attribute.  This function
+    # here is not related to that and implements the full
+    # PyObject_IsInstance() logic.
+
+    # Quick test for an exact match
+    if space.type(w_obj) is w_klass_or_tuple:
+        return True
+
     # -- case (anything, tuple)
     # XXX it might be risky that the JIT sees this
     if space.isinstance_w(w_klass_or_tuple, space.w_tuple):
@@ -58,68 +114,59 @@
         return False
 
     # -- case (anything, type)
-    try:
-        if allow_override:
-            w_result = space.isinstance_allow_override(w_obj, w_klass_or_tuple)
-        else:
-            w_result = space.isinstance(w_obj, w_klass_or_tuple)
-    except OperationError as e:   # if w_klass_or_tuple was not a type, ignore it
-        if not e.match(space, space.w_TypeError):
-            raise       # propagate other errors
-    else:
-        if space.is_true(w_result):
-            return True
-        # From now on we know that w_klass_or_tuple is indeed a type.
-        # Try also to compare it with obj.__class__, if this is not
-        # the same as type(obj).
-        try:
-            w_pretendtype = space.getattr(w_obj, space.wrap('__class__'))
-            if space.is_w(w_pretendtype, space.type(w_obj)):
-                return False     # common case: obj.__class__ is type(obj)
-            if not allow_override:
-                return space.issubtype_w(w_pretendtype, w_klass_or_tuple)
-            w_result = space.issubtype_allow_override(w_pretendtype,
-                                                      w_klass_or_tuple)
-        except OperationError as e:
-            if e.async(space):
-                raise
-            return False      # ignore most exceptions
-        else:
-            return space.is_true(w_result)
+    if allow_override:
+        w_check = space.lookup(w_klass_or_tuple, "__instancecheck__")
+        if w_check is not None:
+            # this is the common case: all type objects have a method
+            # __instancecheck__.  The one in the base 'type' type calls
+            # back p_recursive_isinstance_type_w() from the present module.
+            return space.is_true(space.get_and_call_function(
+                w_check, w_klass_or_tuple, w_obj))
 
-    # -- case (old-style instance, old-style class)
-    if isinstance(w_klass_or_tuple, W_ClassObject):
-        if isinstance(w_obj, W_InstanceObject):
-            return w_obj.w_class.is_subclass_of(w_klass_or_tuple)
-    return _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple)
+    return p_recursive_isinstance_w(space, w_obj, w_klass_or_tuple)
 
 
-def _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple):
-    # -- case (anything, abstract-class)
-    check_class(space, w_klass_or_tuple,
-                "isinstance() arg 2 must be a class, type,"
-                " or tuple of classes and types")
-    try:
-        w_abstractclass = space.getattr(w_obj, space.wrap('__class__'))
-    except OperationError as e:
-        if e.async(space):      # ignore most exceptions
-            raise
-        return False
-    else:
-        return _issubclass_recurse(space, w_abstractclass, w_klass_or_tuple)
+# ---------- issubclass ----------
 
 
 @jit.unroll_safe
-def _issubclass_recurse(space, w_derived, w_top):
-    """Internal helper for abstract cases.  Here, w_top cannot be a tuple."""
-    if space.is_w(w_derived, w_top):
-        return True
-    w_bases = _get_bases(space, w_derived)
-    if w_bases is not None:
-        for w_base in space.fixedview(w_bases):
-            if _issubclass_recurse(space, w_base, w_top):
+def p_abstract_issubclass_w(space, w_derived, w_cls):
+    # Copied straight from CPython 2.7, function abstract_issubclass().
+    # Don't confuse this with the function abstract_issubclass_w() below.
+    # Here, w_cls cannot be a tuple.
+    while True:
+        if space.is_w(w_derived, w_cls):
+            return True
+        w_bases = _get_bases(space, w_derived)
+        if w_bases is None:
+            return False
+        bases_w = space.fixedview(w_bases)
+        last_index = len(bases_w) - 1
+        if last_index < 0:
+            return False
+        # Avoid recursivity in the single inheritance case; in general,
+        # don't recurse on the last item in the tuple (loop instead).
+        for i in range(last_index):
+            if p_abstract_issubclass_w(space, bases_w[i], w_cls):
                 return True
-    return False
+        w_derived = bases_w[last_index]
+
+
+def p_recursive_issubclass_w(space, w_derived, w_cls):
+    # From CPython's function of the same name (which as far as I can tell
+    # is not recursive).  Copied straight from CPython 2.7.
+    if (space.isinstance_w(w_cls, space.w_type) and
+        space.isinstance_w(w_derived, space.w_type)):
+        return space.issubtype_w(w_derived, w_cls)
+    #
+    if (isinstance(w_derived, W_ClassObject) and
+        isinstance(w_cls, W_ClassObject)):
+        return w_derived.is_subclass_of(w_cls)
+    #
+    check_class(space, w_derived, "issubclass() arg 1 must be a class")
+    check_class(space, w_cls, "issubclass() arg 2 must be a class"
+                              " or tuple of classes")
+    return p_abstract_issubclass_w(space, w_derived, w_cls)
 
 
 @jit.unroll_safe
@@ -127,37 +174,31 @@
                           allow_override=False):
     """Implementation for the full 'issubclass(derived, klass_or_tuple)'."""
 
-    # -- case (class-like-object, tuple-of-classes)
+    # WARNING: backward compatibility function name here.  CPython
+    # uses the name "abstract" to refer to the logic of handling
+    # class-like objects, with a "__bases__" attribute.  This function
+    # here is not related to that and implements the full
+    # PyObject_IsSubclass() logic.  There is also p_abstract_issubclass_w().
+
+    # -- case (anything, tuple-of-classes)
     if space.isinstance_w(w_klass_or_tuple, space.w_tuple):
         for w_klass in space.fixedview(w_klass_or_tuple):
             if abstract_issubclass_w(space, w_derived, w_klass, allow_override):
                 return True
         return False
 
-    # -- case (type, type)
-    try:
-        if not allow_override:
-            return space.issubtype_w(w_derived, w_klass_or_tuple)
-        w_result = space.issubtype_allow_override(w_derived, w_klass_or_tuple)
-    except OperationError as e:   # if one of the args was not a type, ignore it
-        if not e.match(space, space.w_TypeError):
-            raise       # propagate other errors
-    else:
-        return space.is_true(w_result)
+    # -- case (anything, type)
+    if allow_override:
+        w_check = space.lookup(w_klass_or_tuple, "__subclasscheck__")

From pypy.commits at gmail.com  Wed Aug 24 14:01:49 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 24 Aug 2016 11:01:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: sys_module.filesystemencoding is not
 wrapped
Message-ID: <57bde10d.c41f1c0a.12d23.8447@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86506:e5e113c82242
Date: 2016-08-24 19:01 +0100
http://bitbucket.org/pypy/pypy/changeset/e5e113c82242/

Log:	sys_module.filesystemencoding is not wrapped

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -144,7 +144,7 @@
             # XXX the two lines above take a few seconds to run whenever
             # we initialize the space; for tests, use a simpler version
             from pypy.module.sys.interp_encoding import base_encoding
-            self.filesystemencoding = space.wrap(base_encoding)
+            self.filesystemencoding = base_encoding
 
     def flush_std_files(self, space):
         w_stdout = space.sys.getdictvalue(space, 'stdout')

From pypy.commits at gmail.com  Wed Aug 24 14:03:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 11:03:40 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fix for the merge
Message-ID: <57bde17c.919a1c0a.c2fea.8156@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86507:290549df34f4
Date: 2016-08-24 19:11 +0100
http://bitbucket.org/pypy/pypy/changeset/290549df34f4/

Log:	fix for the merge

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -171,11 +171,12 @@
 
     # ____________________________________________________________
 
+    extra_return_text = None
+
     def cfunction_body(self):
-        extra_return_text = None
         if self.db.reverse_debugger:
             from rpython.translator.revdb import gencsupp
-            (extra_enter_text, extra_return_text) = (
+            (extra_enter_text, self.extra_return_text) = (
                 gencsupp.prepare_function(self))
             if extra_enter_text:
                 yield extra_enter_text
@@ -219,8 +220,8 @@
                 retval = self.expr(block.inputargs[0])
                 if self.exception_policy != "exc_helper":
                     yield 'RPY_DEBUG_RETURN();'
-                if extra_return_text:
-                    yield extra_return_text
+                if self.extra_return_text:
+                    yield self.extra_return_text
                 yield 'return %s;' % retval
                 return
             elif block.exitswitch is None:

From pypy.commits at gmail.com  Wed Aug 24 14:03:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 11:03:41 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Revert the changes to the
 boehm API: it's still possible to find
Message-ID: <57bde17d.85c11c0a.cb5e1.8890@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86508:91697c0a6b45
Date: 2016-08-24 19:12 +0100
http://bitbucket.org/pypy/pypy/changeset/91697c0a6b45/

Log:	Revert the changes to the boehm API: it's still possible to find
	machines with an old Boehm library installed, e.g. bencher4

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -236,7 +236,7 @@
         if sys.platform == 'win32':
             pass # yield 'assert(GC_all_interior_pointers == 0);'
         else:
-            yield 'GC_set_all_interior_pointers(0);'
+            yield 'GC_all_interior_pointers = 0;'
         yield 'boehm_gc_startup_code();'
 
     def get_real_weakref_type(self):
diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c
--- a/rpython/translator/c/src/mem.c
+++ b/rpython/translator/c/src/mem.c
@@ -117,8 +117,8 @@
 void boehm_gc_startup_code(void)
 {
     GC_init();
-    GC_set_finalizer_notifier(&boehm_gc_finalizer_notifier);
-    GC_set_finalize_on_demand(1);
+    GC_finalizer_notifier = &boehm_gc_finalizer_notifier;
+    GC_finalize_on_demand = 1;
     GC_set_warn_proc(mem_boehm_ignore);
 }
 

From pypy.commits at gmail.com  Wed Aug 24 15:43:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 Aug 2016 12:43:03 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57bdf8c7.d41a1c0a.7323d.cf0e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r779:6b11ddd2bc44
Date: 2016-08-24 21:42 +0200
http://bitbucket.org/pypy/pypy.org/changeset/6b11ddd2bc44/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $64859 of $105000 (61.8%)
+   $64931 of $105000 (61.8%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5336 left
+    we have $5400 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $30875 of $80000 (38.6%)
+   $30891 of $80000 (38.6%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $23528 left
+    we have $23541 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Wed Aug 24 19:07:19 2016
From: pypy.commits at gmail.com (vext01)
Date: Wed, 24 Aug 2016 16:07:19 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Mark a couple of tests
 xfail on OpenBSD (and apply half a fix).
Message-ID: <57be28a7.058a1c0a.a3491.cc10@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86512:e50cf5d2ead3
Date: 2016-08-25 00:02 +0100
http://bitbucket.org/pypy/pypy/changeset/e50cf5d2ead3/

Log:	Mark a couple of tests xfail on OpenBSD (and apply half a fix).

	libdl is not used on OpenBSD for sure, but these tests still fail
	when invoking dlopen() for some reason I can't figure out.

	I'm not sure this is related to W^X but it may be.

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1,4 +1,5 @@
 import py, sys, random, os, struct, operator
+import pytest
 from rpython.jit.metainterp.history import (AbstractFailDescr,
                                          AbstractDescr,
                                          BasicFailDescr,
@@ -2599,6 +2600,8 @@
         deadframe2 = self.cpu.force(frame)
         assert self.cpu.get_int_value(deadframe2, 0) == 30
 
+    @pytest.mark.xfail(sys.platform.startswith("openbsd"),
+                       reason="something wrong with CDLL()")
     def test_call_to_c_function(self):
         from rpython.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL
         from rpython.rtyper.lltypesystem.ll2ctypes import libc_name
@@ -2625,6 +2628,8 @@
         assert fail.identifier == 0
         assert self.cpu.get_int_value(deadframe, 0) == ord('g')
 
+    @pytest.mark.xfail(sys.platform.startswith("openbsd"),
+                       reason="something wrong with CDLL()")
     def test_call_to_c_function_with_callback(self):
         from rpython.rlib.libffi import CDLL, types, ArgChain, clibffi
         from rpython.rtyper.lltypesystem.ll2ctypes import libc_name
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -17,6 +17,7 @@
 _MAC_OS = platform.name == "darwin"
 _FREEBSD = sys.platform.startswith("freebsd")
 _NETBSD = sys.platform.startswith("netbsd")
+_OPENBSD = sys.platform.startswith("openbsd")
 
 if _WIN32:
     from rpython.rlib import rwin32
@@ -29,7 +30,7 @@
 else:
     pre_include_bits = []
 
-if _FREEBSD or _NETBSD or _WIN32:
+if _OPENBSD or _FREEBSD or _NETBSD or _WIN32:
     libraries = []
 else:
     libraries = ['dl']

From pypy.commits at gmail.com  Wed Aug 24 19:07:14 2016
From: pypy.commits at gmail.com (vext01)
Date: Wed, 24 Aug 2016 16:07:14 -0700 (PDT)
Subject: [pypy-commit] pypy w-xor-x2: Allocate mmap pages with PROT_NONE and
 provide ability to use mprotect.
Message-ID: <57be28a2.465d1c0a.3dddb.e520@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: w-xor-x2
Changeset: r86509:027b44340648
Date: 2016-08-24 23:43 +0100
http://bitbucket.org/pypy/pypy/changeset/027b44340648/

Log:	Allocate mmap pages with PROT_NONE and provide ability to use
	mprotect.

diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -57,7 +57,7 @@
     # some constants are linux only so they will be correctly exposed outside
     # depending on the OS
     constant_names = ['MAP_SHARED', 'MAP_PRIVATE', 'MAP_FIXED',
-                      'PROT_READ', 'PROT_WRITE',
+                      'PROT_READ', 'PROT_WRITE', 'PROT_NONE',
                       'MS_SYNC']
     opt_constant_names = ['MAP_ANON', 'MAP_ANONYMOUS', 'MAP_NORESERVE',
                           'PROT_EXEC',
@@ -155,6 +155,10 @@
     c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT,
                                    rffi.INT, off_t], PTR, macro=True,
                                    save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO)
+
+    _, c_mprotect_safe = external('mprotect', [PTR, size_t, rffi.INT],
+                                  rffi.INT, _nowrapper=True)
+
     # 'mmap' on linux32 is a macro that calls 'mmap64'
     _, c_munmap_safe = external('munmap', [PTR, size_t], rffi.INT)
     c_msync, _ = external('msync', [PTR, size_t, rffi.INT], rffi.INT,
@@ -172,6 +176,8 @@
     _get_allocation_granularity = _get_page_size = lambda: _pagesize
 
 elif _MS_WINDOWS:
+    # XXX mprotect equivalent
+    # XXX set_pages_executable/writable equivalent
 
     class ComplexCConfig:
         _compilation_info_ = CConfig._compilation_info_
@@ -707,12 +713,29 @@
 
     def alloc_hinted(hintp, map_size):
         flags = MAP_PRIVATE | MAP_ANONYMOUS
-        prot = PROT_EXEC | PROT_READ | PROT_WRITE
+        # Pages start with no permissions
+        prot = PROT_NONE
         if we_are_translated():
             flags = NonConstant(flags)
             prot = NonConstant(prot)
         return c_mmap_safe(hintp, map_size, prot, flags, -1, 0)
 
+    def mprotect(addr, size, prot):
+        addr = rffi.cast(PTR, addr)
+        size = rffi.cast(lltype.Unsigned, size)
+        prot = rffi.cast(rffi.INT, prot)
+        return c_mprotect_safe(addr, size, prot)
+
+    def set_pages_executable(addr, size):
+        rv = mprotect(addr, size, PROT_EXEC | PROT_READ)
+        if rv < 0:
+            debug.fatalerror_notb("set_pages_executable failed")
+
+    def set_pages_writable(addr, size):
+        rv = mprotect(addr, size, PROT_WRITE | PROT_READ)
+        if rv < 0:
+            debug.fatalerror_notb("set_pages_executable failed")
+
     def clear_large_memory_chunk_aligned(addr, map_size):
         addr = rffi.cast(PTR, addr)
         flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS

From pypy.commits at gmail.com  Wed Aug 24 19:07:21 2016
From: pypy.commits at gmail.com (vext01)
Date: Wed, 24 Aug 2016 16:07:21 -0700 (PDT)
Subject: [pypy-commit] pypy w-xor-x2: Don't use this branch.
Message-ID: <57be28a9.6211c20a.7af0d.b01c@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: w-xor-x2
Changeset: r86513:faca61b444b4
Date: 2016-08-25 00:05 +0100
http://bitbucket.org/pypy/pypy/changeset/faca61b444b4/

Log:	Don't use this branch.


From pypy.commits at gmail.com  Wed Aug 24 19:07:16 2016
From: pypy.commits at gmail.com (vext01)
Date: Wed, 24 Aug 2016 16:07:16 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Allocate mmap pages
 with PROT_NONE and provide ability to use mprotect.
Message-ID: <57be28a4.271ac20a.51b3b.a697@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86510:1425d9b5d850
Date: 2016-08-24 23:43 +0100
http://bitbucket.org/pypy/pypy/changeset/1425d9b5d850/

Log:	Allocate mmap pages with PROT_NONE and provide ability to use
	mprotect.

diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -57,7 +57,7 @@
     # some constants are linux only so they will be correctly exposed outside
     # depending on the OS
     constant_names = ['MAP_SHARED', 'MAP_PRIVATE', 'MAP_FIXED',
-                      'PROT_READ', 'PROT_WRITE',
+                      'PROT_READ', 'PROT_WRITE', 'PROT_NONE',
                       'MS_SYNC']
     opt_constant_names = ['MAP_ANON', 'MAP_ANONYMOUS', 'MAP_NORESERVE',
                           'PROT_EXEC',
@@ -155,6 +155,10 @@
     c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT,
                                    rffi.INT, off_t], PTR, macro=True,
                                    save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO)
+
+    _, c_mprotect_safe = external('mprotect', [PTR, size_t, rffi.INT],
+                                  rffi.INT, _nowrapper=True)
+
     # 'mmap' on linux32 is a macro that calls 'mmap64'
     _, c_munmap_safe = external('munmap', [PTR, size_t], rffi.INT)
     c_msync, _ = external('msync', [PTR, size_t, rffi.INT], rffi.INT,
@@ -172,6 +176,8 @@
     _get_allocation_granularity = _get_page_size = lambda: _pagesize
 
 elif _MS_WINDOWS:
+    # XXX mprotect equivalent
+    # XXX set_pages_executable/writable equivalent
 
     class ComplexCConfig:
         _compilation_info_ = CConfig._compilation_info_
@@ -707,12 +713,29 @@
 
     def alloc_hinted(hintp, map_size):
         flags = MAP_PRIVATE | MAP_ANONYMOUS
-        prot = PROT_EXEC | PROT_READ | PROT_WRITE
+        # Pages start with no permissions
+        prot = PROT_NONE
         if we_are_translated():
             flags = NonConstant(flags)
             prot = NonConstant(prot)
         return c_mmap_safe(hintp, map_size, prot, flags, -1, 0)
 
+    def mprotect(addr, size, prot):
+        addr = rffi.cast(PTR, addr)
+        size = rffi.cast(lltype.Unsigned, size)
+        prot = rffi.cast(rffi.INT, prot)
+        return c_mprotect_safe(addr, size, prot)
+
+    def set_pages_executable(addr, size):
+        rv = mprotect(addr, size, PROT_EXEC | PROT_READ)
+        if rv < 0:
+            debug.fatalerror_notb("set_pages_executable failed")
+
+    def set_pages_writable(addr, size):
+        rv = mprotect(addr, size, PROT_WRITE | PROT_READ)
+        if rv < 0:
+            debug.fatalerror_notb("set_pages_executable failed")
+
     def clear_large_memory_chunk_aligned(addr, map_size):
         addr = rffi.cast(PTR, addr)
         flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS

From pypy.commits at gmail.com  Wed Aug 24 19:27:16 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 24 Aug 2016 16:27:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: update version numbers for release
Message-ID: <57be2d54.a111c20a.54f43.af07@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86514:531050b1f410
Date: 2016-08-25 08:59 +1000
http://bitbucket.org/pypy/pypy/changeset/531050b1f410/

Log:	update version numbers for release

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
rename from pypy/doc/whatsnew-head.rst
rename to pypy/doc/whatsnew-pypy2-5.4.0.rst
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.4.1-alpha0"
+#define PYPY_VERSION_NUM  0x05040100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 2, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Wed Aug 24 19:27:18 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 24 Aug 2016 16:27:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: restart whatsnew
Message-ID: <57be2d56.6974c20a.8a0d2.af66@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86515:9734e4f070e5
Date: 2016-08-25 09:00 +1000
http://bitbucket.org/pypy/pypy/changeset/9734e4f070e5/

Log:	restart whatsnew

diff --git a/pypy/doc/whatnew-head.rst b/pypy/doc/whatnew-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatnew-head.rst
@@ -0,0 +1,8 @@
+==========================
+What's new in PyPy2.7 5.4+
+==========================
+
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 531050b1f410
+
+

From pypy.commits at gmail.com  Wed Aug 24 19:27:21 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 24 Aug 2016 16:27:21 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: merge default into release branch
Message-ID: <57be2d59.436ec20a.d0586.b3ac@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86516:b637e157f241
Date: 2016-08-25 09:03 +1000
http://bitbucket.org/pypy/pypy/changeset/b637e157f241/

Log:	merge default into release branch

diff too long, truncating to 2000 out of 25848 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -85,10 +85,11 @@
     pass
 
 def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')):
+    import ast
     result = []
     for word in re_word.findall(line):
         if word.startswith('"'):
-            word = eval(word)
+            word = ast.literal_eval(word)
         result.append(word)
     return result
 
diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib-python/2.7/test/test_hash.py b/lib-python/2.7/test/test_hash.py
--- a/lib-python/2.7/test/test_hash.py
+++ b/lib-python/2.7/test/test_hash.py
@@ -174,7 +174,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
         else:
             return self.value
 
-    def __buffer__(self):
+    def __buffer__(self, flags):
         return buffer(self._buffer)
 
     def _get_b_base(self):
@@ -199,10 +199,13 @@
     return tp._alignmentofinstances()
 
 @builtinify
-def byref(cdata):
+def byref(cdata, offset=0):
     # "pointer" is imported at the end of this module to avoid circular
     # imports
-    return pointer(cdata)
+    ptr = pointer(cdata)
+    if offset != 0:
+        ptr._buffer[0] += offset
+    return ptr
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -0,0 +1,91 @@
+# Note: uses the CFFI out-of-line ABI mode.  We can't use the API
+# mode because ffi.compile() needs to run the compiler, which
+# needs 'subprocess', which needs 'msvcrt' and '_subprocess',
+# which depend on '_pypy_winbase_cffi' already.
+#
+# Note that if you need to regenerate _pypy_winbase_cffi and
+# can't use a preexisting PyPy to do that, then running this
+# file should work as long as 'subprocess' is not imported
+# by cffi.  I had to hack in 'cffi._pycparser' to move an
+#'import subprocess' to the inside of a function.  (Also,
+# CPython+CFFI should work as well.)
+#
+# This module supports both msvcrt.py and _subprocess.py.
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pypy_winbase_cffi", None)
+
+# ---------- MSVCRT ----------
+
+ffi.cdef("""
+typedef unsigned short wint_t;
+
+int _open_osfhandle(intptr_t osfhandle, int flags);
+intptr_t _get_osfhandle(int fd);
+int _setmode(int fd, int mode);
+int _locking(int fd, int mode, long nbytes);
+
+int _kbhit(void);
+int _getch(void);
+wint_t _getwch(void);
+int _getche(void);
+wint_t _getwche(void);
+int _putch(int);
+wint_t _putwch(wchar_t);
+int _ungetch(int);
+wint_t _ungetwch(wint_t);
+""")
+
+# ---------- SUBPROCESS ----------
+
+ffi.cdef("""
+typedef struct {
+    DWORD  cb;
+    char * lpReserved;
+    char * lpDesktop;
+    char * lpTitle;
+    DWORD  dwX;
+    DWORD  dwY;
+    DWORD  dwXSize;
+    DWORD  dwYSize;
+    DWORD  dwXCountChars;
+    DWORD  dwYCountChars;
+    DWORD  dwFillAttribute;
+    DWORD  dwFlags;
+    WORD   wShowWindow;
+    WORD   cbReserved2;
+    LPBYTE lpReserved2;
+    HANDLE hStdInput;
+    HANDLE hStdOutput;
+    HANDLE hStdError;
+} STARTUPINFO, *LPSTARTUPINFO;
+
+typedef struct {
+    HANDLE hProcess;
+    HANDLE hThread;
+    DWORD  dwProcessId;
+    DWORD  dwThreadId;
+} PROCESS_INFORMATION, *LPPROCESS_INFORMATION;
+
+DWORD WINAPI GetVersion(void);
+BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD);
+BOOL WINAPI CloseHandle(HANDLE);
+HANDLE WINAPI GetCurrentProcess(void);
+BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
+                            DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
+BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
+BOOL WINAPI TerminateProcess(HANDLE, UINT);
+HANDLE WINAPI GetStdHandle(DWORD);
+""")
+
+# --------------------
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -0,0 +1,10 @@
+# auto-generated file
+import _cffi_backend
+
+ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
+    _version = 0x2601,
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+)
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -10,148 +10,99 @@
 
 # Declare external Win32 functions
 
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
+from _pypy_winbase_cffi import ffi as _ffi
+_kernel32 = _ffi.dlopen('kernel32')
 
 GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
 
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
-
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
-
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
-
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
-
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessA
-_CreateProcess.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
 
 # Now the _subprocess module implementation
 
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
+def _WinError():
+    code, message = _ffi.getwinerror()
+    raise WindowsError(code, message)
 
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
+_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+
+class _handle(object):
+    def __init__(self, c_handle):
+        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
+        self.c_handle = c_handle
+        if int(self) != -1:
+            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
 
     def __int__(self):
-        return self.handle
+        return int(_ffi.cast("intptr_t", self.c_handle))
 
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
+    def __repr__(self):
+        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
 
     def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
+        h = int(self)
+        if h != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+        return h
 
     def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
+        if int(self) != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+            _kernel32.CloseHandle(c_handle)
 
 def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
+    handles = _ffi.new("HANDLE[2]")
 
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
+    res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size)
 
     if not res:
         raise _WinError()
 
-    return _handle(read.value), _handle(write.value)
+    return _handle(handles[0]), _handle(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
+    return _handle(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
+    # CPython: the first three arguments are expected to be integers
+    target = _ffi.new("HANDLE[1]")
 
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
+    res = _kernel32.DuplicateHandle(
+        _ffi.cast("HANDLE", source_process),
+        _ffi.cast("HANDLE", source),
+        _ffi.cast("HANDLE", target_process),
+        target, access, inherit, options)
 
     if not res:
         raise _WinError()
 
-    return _handle(target.value)
+    return _handle(target[0])
+
+def _z(input):
+    if input is None:
+        return _ffi.NULL
+    if isinstance(input, basestring):
+        return str(input)
+    raise TypeError("string/unicode/None expected, got %r" % (
+        type(input).__name__,))
 
 def CreateProcess(name, command_line, process_attr, thread_attr,
                   inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
+    si = _ffi.new("STARTUPINFO *")
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
+        # CPython: these three handles are expected to be _handle objects
         if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
+            si.hStdInput = startup_info.hStdInput.c_handle
         if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
+            si.hStdOutput = startup_info.hStdOutput.c_handle
         if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
+            si.hStdError = startup_info.hStdError.c_handle
 
-    pi = _PROCESS_INFORMATION()
+    pi = _ffi.new("PROCESS_INFORMATION *")
 
     if env is not None:
         envbuf = ""
@@ -159,47 +110,55 @@
             envbuf += "%s=%s\0" % (k, v)
         envbuf += '\0'
     else:
-        envbuf = None
+        envbuf = _ffi.NULL
 
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
+    res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL,
+                                   _ffi.NULL, inherit, flags, envbuf,
+                                   _z(start_dir), si, pi)
 
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
 
 def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
+    # CPython: the first argument is expected to be an integer.
+    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
+                                        milliseconds)
     if res < 0:
         raise _WinError()
 
     return res
 
 def GetExitCodeProcess(handle):
-    code = _c_int()
+    # CPython: the first argument is expected to be an integer.
+    code = _ffi.new("DWORD[1]")
 
-    res = _GetExitCodeProcess(int(handle), _byref(code))
+    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
 
     if not res:
         raise _WinError()
 
-    return code.value
+    return code[0]
 
 def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
+    # CPython: the first argument is expected to be an integer.
+    # The second argument is silently wrapped in a UINT.
+    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+                                     _ffi.cast("UINT", exitcode))
 
     if not res:
         raise _WinError()
 
 def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
+    stdhandle = _ffi.cast("DWORD", stdhandle)
+    res = _kernel32.GetStdHandle(stdhandle)
 
     if not res:
         return None
     else:
-        return res
+        # note: returns integer, not handle object
+        return int(_ffi.cast("intptr_t", res))
 
 STD_INPUT_HANDLE = -10
 STD_OUTPUT_HANDLE = -11
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py
--- a/lib_pypy/cffi/_pycparser/__init__.py
+++ b/lib_pypy/cffi/_pycparser/__init__.py
@@ -10,7 +10,6 @@
 __all__ = ['c_lexer', 'c_parser', 'c_ast']
 __version__ = '2.14'
 
-from subprocess import Popen, PIPE
 from .c_parser import CParser
 
 
@@ -28,6 +27,7 @@
         When successful, returns the preprocessed file's contents.
         Errors from cpp will be printed out.
     """
+    from subprocess import Popen, PIPE
     path_list = [cpp_path]
     if isinstance(cpp_args, list):
         path_list += cpp_args
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -513,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -570,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -839,7 +839,7 @@
             month = self._month
         if day is None:
             day = self._day
-        return date(year, month, day)
+        return date.__new__(type(self), year, month, day)
 
     # Comparisons of date objects with other.
 
@@ -1356,7 +1356,8 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return time(hour, minute, second, microsecond, tzinfo)
+        return time.__new__(type(self),
+                            hour, minute, second, microsecond, tzinfo)
 
     def __nonzero__(self):
         if self.second or self.microsecond:
@@ -1566,8 +1567,9 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return datetime(year, month, day, hour, minute, second, microsecond,
-                        tzinfo)
+        return datetime.__new__(type(self),
+                                year, month, day, hour, minute, second,
+                                microsecond, tzinfo)
 
     def astimezone(self, tz):
         if not isinstance(tz, tzinfo):
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -7,26 +7,39 @@
 # XXX incomplete: implemented only functions needed by subprocess.py
 # PAC: 2010/08 added MS locking for Whoosh
 
-import ctypes
+# 07/2016: rewrote in CFFI
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The 'msvcrt' module is only available on Windows")
+
+import _rawffi
+from _pypy_winbase_cffi import ffi as _ffi
+_lib = _ffi.dlopen(_rawffi.get_libc().name)
+
 import errno
-from ctypes_support import standard_c_lib as _c
-from ctypes_support import get_errno
-
-try:
-    open_osfhandle = _c._open_osfhandle
-except AttributeError: # we are not on windows
-    raise ImportError
 
 try: from __pypy__ import builtinify, validate_fd
 except ImportError: builtinify = validate_fd = lambda f: f
 
 
-open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int]
-open_osfhandle.restype = ctypes.c_int
+def _ioerr():
+    e = _ffi.errno
+    raise IOError(e, errno.errorcode[e])
 
-_get_osfhandle = _c._get_osfhandle
-_get_osfhandle.argtypes = [ctypes.c_int]
-_get_osfhandle.restype = ctypes.c_int
+
+ at builtinify
+def open_osfhandle(fd, flags):
+    """"open_osfhandle(handle, flags) -> file descriptor
+
+    Create a C runtime file descriptor from the file handle handle. The
+    flags parameter should be a bitwise OR of os.O_APPEND, os.O_RDONLY,
+    and os.O_TEXT. The returned file descriptor may be used as a parameter
+    to os.fdopen() to create a file object."""
+    fd = _lib._open_osfhandle(fd, flags)
+    if fd == -1:
+        _ioerr()
+    return fd
 
 @builtinify
 def get_osfhandle(fd):
@@ -38,62 +51,74 @@
         validate_fd(fd)
     except OSError as e:
         raise IOError(*e.args)
-    return _get_osfhandle(fd)
+    result = _lib._get_osfhandle(fd)
+    if result == -1:
+        _ioerr()
+    return result
 
-setmode = _c._setmode
-setmode.argtypes = [ctypes.c_int, ctypes.c_int]
-setmode.restype = ctypes.c_int
+ at builtinify
+def setmode(fd, flags):
+    """setmode(fd, mode) -> Previous mode
+
+    Set the line-end translation mode for the file descriptor fd. To set
+    it to text mode, flags should be os.O_TEXT; for binary, it should be
+    os.O_BINARY."""
+    flags = _lib._setmode(fd, flags)
+    if flags == -1:
+        _ioerr()
+    return flags
 
 LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5)
 
-_locking = _c._locking
-_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_locking.restype = ctypes.c_int
-
 @builtinify
 def locking(fd, mode, nbytes):
-    '''lock or unlock a number of bytes in a file.'''
-    rv = _locking(fd, mode, nbytes)
+    """"locking(fd, mode, nbytes) -> None
+
+    Lock part of a file based on file descriptor fd from the C runtime.
+    Raises IOError on failure. The locked region of the file extends from
+    the current file position for nbytes bytes, and may continue beyond
+    the end of the file. mode must be one of the LK_* constants listed
+    below. Multiple regions in a file may be locked at the same time, but
+    may not overlap. Adjacent regions are not merged; they must be unlocked
+    individually."""
+    rv = _lib._locking(fd, mode, nbytes)
     if rv != 0:
-        e = get_errno()
-        raise IOError(e, errno.errorcode[e])
+        _ioerr()
 
 # Console I/O routines
 
-kbhit = _c._kbhit
-kbhit.argtypes = []
-kbhit.restype = ctypes.c_int
+kbhit = _lib._kbhit
 
-getch = _c._getch
-getch.argtypes = []
-getch.restype = ctypes.c_char
+ at builtinify
+def getch():
+    return chr(_lib._getch())
 
-getwch = _c._getwch
-getwch.argtypes = []
-getwch.restype = ctypes.c_wchar
+ at builtinify
+def getwch():
+    return unichr(_lib._getwch())
 
-getche = _c._getche
-getche.argtypes = []
-getche.restype = ctypes.c_char
+ at builtinify
+def getche():
+    return chr(_lib._getche())
 
-getwche = _c._getwche
-getwche.argtypes = []
-getwche.restype = ctypes.c_wchar
+ at builtinify
+def getwche():
+    return unichr(_lib._getwche())
 
-putch = _c._putch
-putch.argtypes = [ctypes.c_char]
-putch.restype = None
+ at builtinify
+def putch(ch):
+    _lib._putch(ord(ch))
 
-putwch = _c._putwch
-putwch.argtypes = [ctypes.c_wchar]
-putwch.restype = None
+ at builtinify
+def putwch(ch):
+    _lib._putwch(ord(ch))
 
-ungetch = _c._ungetch
-ungetch.argtypes = [ctypes.c_char]
-ungetch.restype = None
+ at builtinify
+def ungetch(ch):
+    if _lib._ungetch(ord(ch)) == -1:   # EOF
+        _ioerr()
 
-ungetwch = _c._ungetwch
-ungetwch.argtypes = [ctypes.c_wchar]
-ungetwch.restype = None
-
-del ctypes
+ at builtinify
+def ungetwch(ch):
+    if _lib._ungetwch(ord(ch)) == -1:   # EOF
+        _ioerr()
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -36,7 +36,7 @@
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
-    "_csv", "cppyy", "_pypyjson",
+    "_csv", "cppyy", "_pypyjson", "_jitlog"
 ])
 
 from rpython.jit.backend import detect_cpu
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -76,6 +76,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -110,9 +124,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -120,11 +131,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -153,35 +160,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
-def pytest_runtest_teardown(__multicall__, item):
-    __multicall__.execute()
-
-    if 'pygame' in sys.modules:
-        assert option.view, ("should not invoke Pygame "
-                             "if conftest.option.view is False")
-
-
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
 
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -104,27 +104,24 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev liblzma-dev
-
-For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+    tk-dev libgc-dev \
+    liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel
-
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+    gdbm-devel \
+    xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses
+    libexpat-devel libffi-devel python-curses \
+    xz-devel  # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
-
 On Mac OS X, most of these build-time dependencies are installed alongside
 the Developer Tools. However, note that in order for the installation to
 find them you may need to run::
diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt
--- a/pypy/doc/config/commandline.txt
+++ b/pypy/doc/config/commandline.txt
@@ -9,7 +9,7 @@
 PyPy Python interpreter options
 -------------------------------
 
-The following options can be used after ``translate.py
+The following options can be used after ``rpython
 targetpypystandalone`` or as options to ``py.py``.
 
 .. GENERATE: objspace
@@ -22,7 +22,7 @@
 General translation options
 ---------------------------
 
-The following are options of ``translate.py``.  They must be
+The following are options of ``bin/rpython``.  They must be
 given before the ``targetxxx`` on the command line.
 
 * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]`
diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst
--- a/pypy/doc/config/index.rst
+++ b/pypy/doc/config/index.rst
@@ -15,12 +15,12 @@
 
     ./py.py <`objspace options`_>
 
-and the ``translate.py`` translation entry
+and the ``rpython/bin/rpython`` translation entry
 point which takes arguments of this form:
 
 .. parsed-literal::
 
-    ./translate.py <`translation options`_> <target>
+    ./rpython/bin/rpython <`translation options`_> <target>
 
 For the common case of ``<target>`` being ``targetpypystandalone.py``,
 you can then pass the `object space options`_ after
@@ -28,7 +28,7 @@
 
 .. parsed-literal::
 
-    ./translate.py <`translation options`_> targetpypystandalone.py <`objspace options`_>
+    ./rpython/bin/rpython <`translation options`_> targetpypystandalone.py <`objspace options`_>
 
 There is an `overview`_ of all command line arguments that can be
 passed in either position.
diff --git a/pypy/doc/config/opt.rst b/pypy/doc/config/opt.rst
--- a/pypy/doc/config/opt.rst
+++ b/pypy/doc/config/opt.rst
@@ -4,8 +4,8 @@
 This meta-option selects a default set of optimization
 settings to use during a translation.  Usage::
 
-    translate.py --opt=#
-    translate.py -O#
+    bin/rpython --opt=#
+    bin/rpython -O#
 
 where ``#`` is the desired optimization level.  The valid choices are:
 
diff --git a/pypy/doc/config/translation.dont_write_c_files.txt b/pypy/doc/config/translation.dont_write_c_files.txt
--- a/pypy/doc/config/translation.dont_write_c_files.txt
+++ b/pypy/doc/config/translation.dont_write_c_files.txt
@@ -1,4 +1,4 @@
 write the generated C files to ``/dev/null`` instead of to the disk. Useful if
-you want to use translate.py as a benchmark and don't want to access the disk.
+you want to use translation as a benchmark and don't want to access the disk.
 
 .. _`translation documentation`: ../translation.html
diff --git a/pypy/doc/config/translation.fork_before.txt b/pypy/doc/config/translation.fork_before.txt
--- a/pypy/doc/config/translation.fork_before.txt
+++ b/pypy/doc/config/translation.fork_before.txt
@@ -1,4 +1,4 @@
 This is an option mostly useful when working on the PyPy toolchain. If you use
-it, translate.py will fork before the specified phase. If the translation
+it, translation will fork before the specified phase. If the translation
 crashes after that fork, you can fix the bug in the toolchain, and continue
 translation at the fork-point.
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -122,7 +122,7 @@
     $ hg up reflex-support         # optional
 
     # This example shows python, but using pypy-c is faster and uses less memory
-    $ python rpython/translator/goal/translate.py --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy
+    $ python rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy
 
 This will build a ``pypy-c`` that includes the cppyy module, and through that,
 Reflex support.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
@@ -315,13 +322,28 @@
 
  - ``complex``
 
+ - ``str`` (empty or single-character strings only)
+
+ - ``unicode`` (empty or single-character strings only)
+
+ - ``tuple`` (empty tuples only)
+
+ - ``frozenset`` (empty frozenset only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
-Notably missing from the list above are ``str`` and ``unicode``.  If your
-code relies on comparing strings with ``is``, then it might break in PyPy.
+Note that strings of length 2 or greater can be equal without being
+identical.  Similarly, ``x is (2,)`` is not necessarily true even if
+``x`` contains a tuple and ``x == (2,)``.  The uniqueness rules apply
+only to the particular cases described above.  The ``str``, ``unicode``,
+``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a
+test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was
+equal to ``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is
+closer to CPython's, which caches precisely the empty tuple/frozenset,
+and (generally but not always) the strings and unicodes of length <= 1.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -335,3 +335,65 @@
 
 This will disable SELinux's protection and allow PyPy to configure correctly.
 Be sure to enable it again if you need it!
+
+
+How should I report a bug?
+--------------------------
+
+Our bug tracker is here: https://bitbucket.org/pypy/pypy/issues/
+
+Missing features or incompatibilities with CPython are considered
+bugs, and they are welcome.  (See also our list of `known
+incompatibilities`__.)
+
+.. __: http://pypy.org/compat.html
+
+For bugs of the kind "I'm getting a PyPy crash or a strange
+exception", please note that: **We can't do anything without
+reproducing the bug ourselves**.  We cannot do anything with
+tracebacks from gdb, or core dumps.  This is not only because the
+standard PyPy is compiled without debug symbols.  The real reason is
+that a C-level traceback is usually of no help at all in PyPy.
+Debugging PyPy can be annoying.
+
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
+In more details:
+
+* First, please give the exact PyPy version, and the OS.
+
+* It might help focus our search if we know if the bug can be
+  reproduced on a "``pypy --jit off``" or not.  If "``pypy --jit
+  off``" always works, then the problem might be in the JIT.
+  Otherwise, we know we can ignore that part.
+
+* If you got the bug using only Open Source components, please give a
+  step-by-step guide that we can follow to reproduce the problem
+  ourselves.  Don't assume we know anything about any program other
+  than PyPy.  We would like a guide that we can follow point by point
+  (without guessing or having to figure things out)
+  on a machine similar to yours, starting from a bare PyPy, until we
+  see the same problem.  (If you can, you can try to reduce the number
+  of steps and the time it needs to run, but that is not mandatory.)
+
+* If the bug involves Closed Source components, or just too many Open
+  Source components to install them all ourselves, then maybe you can
+  give us some temporary ssh access to a machine where the bug can be
+  reproduced.  Or, maybe we can download a VirtualBox or VMWare
+  virtual machine where the problem occurs.
+
+* If giving us access would require us to use tools other than ssh,
+  make appointments, or sign a NDA, then we can consider a commerical
+  support contract for a small sum of money.
+
+* If even that is not possible for you, then sorry, we can't help.
+
+Of course, you can try to debug the problem yourself, and we can help
+you get started if you ask on the #pypy IRC channel, but be prepared:
+debugging an annoying PyPy problem usually involves quite a lot of gdb
+in auto-generated C code, and at least some knowledge about the
+various components involved, from PyPy's own RPython source code to
+the GC and possibly the JIT.
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,8 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
+   release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -39,17 +39,16 @@
 library.
 
 If you want to install 3rd party libraries, the most convenient way is
-to install pip_ (unless you want to install virtualenv as explained
-below; then you can directly use pip inside virtualenvs):
+to install pip_ using ensurepip_ (unless you want to install virtualenv as 
+explained below; then you can directly use pip inside virtualenvs):
 
 .. code-block:: console
 
-    $ curl -O https://bootstrap.pypa.io/get-pip.py
-    $ ./pypy-2.1/bin/pypy get-pip.py
-    $ ./pypy-2.1/bin/pip install pygments  # for example
+    $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install pygments  # for example
 
-Third party libraries will be installed in ``pypy-2.1/site-packages``, and
-the scripts in ``pypy-2.1/bin``.
+Third party libraries will be installed in ``pypy-xxx/site-packages``, and
+the scripts in ``pypy-xxx/bin``.
 
 
 Installing using virtualenv
@@ -61,7 +60,7 @@
 checkout::
 
 	# from a tarball
-	$ virtualenv -p /opt/pypy-c-jit-41718-3fb486695f20-linux/bin/pypy my-pypy-env
+	$ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env
 
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
@@ -69,7 +68,7 @@
 Note that bin/python is now a symlink to bin/pypy.
 
 .. _pip: http://pypi.python.org/pypi/pip
-
+.. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html
 
 Building PyPy yourself
 ~~~~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst
--- a/pypy/doc/release-pypy2.7-v5.3.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.3.0.rst
@@ -176,8 +176,8 @@
   * Reduce the size of generated code by using the same function objects in
     all generated subclasses
 
- * Share cpyext Py* function wrappers according to the signature, shrining the
-   translated libpypy.so by about 
+  * Share cpyext Py* function wrappers according to the signature, shrinking the
+    translated libpypy.so by about 10% (measured without the JIT)
 
   * Compile c snippets with -Werror, and fix warnings it exposed
 
diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -0,0 +1,41 @@
+==========
+PyPy 5.3.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.3.0, released last week,
+due to issues_ reported by users.
+
+Thanks to those who reported the issues.
+
+.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,142 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+and fixed many issues and bugs raised by the growing community of PyPy
+users.
+
+XXXXX MORE ???
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix OpenBSD build breakage and support OpenBSD in VMProf.
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatnew-head.rst b/pypy/doc/whatnew-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatnew-head.rst
@@ -0,0 +1,8 @@
+==========================
+What's new in PyPy2.7 5.4+
+==========================
+
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 531050b1f410
+
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-head.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-=========================
-What's new in PyPy2.7 5.3+
-=========================
-
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
-
diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -0,0 +1,15 @@
+===========================
+What's new in PyPy2.7 5.3.1
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3.0
+.. startrev: f4d726d1a010
+
+
+A bug-fix release, merging these changes:
+
+  * Add include guards to pymem.h, fixes issue #2321
+
+  * Make vmprof build on OpenBSD, from pull request #456
+
+  * Fix ``bytearray('').replace('a', 'ab')``, issue #2324
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -0,0 +1,161 @@
+==========================
+What's new in PyPy2.7 5.3+
+==========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
+
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -24,11 +24,15 @@
 -V     : print the Python version number and exit (also --version)
 -W arg : warning control; arg is action:message:category:module:lineno
          also PYTHONWARNINGS=arg
+-X arg : set implementation-specific option
 file   : program read from script file
 -      : program read from stdin (default; interactive mode if a tty)
 arg ...: arguments passed to program in sys.argv[1:]
+
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
+-X track-resources : track the creation of files and sockets and display
+                     a warning if they are not closed explicitly
 """
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
@@ -229,6 +233,14 @@
         import pypyjit
         pypyjit.set_param(jitparam)
 
+def set_runtime_options(options, Xparam, *args):
+    if Xparam == 'track-resources':
+        sys.pypy_set_track_resources(True)
+    else:
+        print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),)
+        print >> sys.stderr, '[options] can be: track-resources'
+        raise SystemExit
+
 class CommandLineError(Exception):
     pass
 
@@ -404,6 +416,7 @@
     '--info':    (print_info,      None),
     '--jit':     (set_jit_option,  Ellipsis),
     '-funroll-loops': (funroll_loops, None),
+    '-X':        (set_runtime_options, Ellipsis),
     '--':        (end_options,     None),
     }
 
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -387,7 +387,8 @@
     def _stacksize(self, blocks):
         """Compute co_stacksize."""
         for block in blocks:
-            block.initial_depth = 0
+            block.initial_depth = -99
+        blocks[0].initial_depth = 0
         # Assumes that it is sufficient to walk the blocks in 'post-order'.
         # This means we ignore all back-edges, but apart from that, we only
         # look into a block when all the previous blocks have been done.
@@ -406,8 +407,11 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
+        if depth == -99:     # this block is never reached, skip
+             return 0
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
+            assert depth >= 0
             if depth >= self._max_depth:
                 self._max_depth = depth
             jump_op = instr.opcode
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -108,8 +108,15 @@
         return getattr(space, name)(operand)
     return do_fold
 
-def _fold_pow(space, left, right):
-    return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+    # don't constant-fold if "w_left" and "w_right" are integers and
+    # the estimated bit length of the power is unreasonably large
+    space.appexec([w_left, w_right], """(left, right):
+        if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+            if left.bit_length() * right > 5000:
+                raise OverflowError
+    """)
+    return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
     return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -458,14 +458,17 @@
         decl = str(decl) + "\n"
         yield self.st, decl, 'x', (1, 2, 3, 4)
 
+    def test_closure_error(self):
         source = """if 1:
         def f(a):
             del a
             def x():
                 a
         """
-        exc = py.test.raises(SyntaxError, self.run, source).value
-        assert exc.msg == "Can't delete variable used in nested scopes: 'a'"
+        with py.test.raises(SyntaxError) as excinfo:
+            self.run(source)
+        msg = excinfo.value.msg
+        assert msg == "Can't delete variable used in nested scopes: 'a'"
 
     def test_try_except_finally(self):
         yield self.simple_test, """
@@ -879,7 +882,20 @@
         """
         self.simple_test(source, 'ok', 1)
 
-    def test_remove_docstring(self):
+    @py.test.mark.parametrize('expr, result', [
+        ("f1.__doc__", None),
+        ("f2.__doc__", 'docstring'),
+        ("f2()", 'docstring'),
+        ("f3.__doc__", None),
+        ("f3()", 'bar'),
+        ("C1.__doc__", None),
+        ("C2.__doc__", 'docstring'),
+        ("C3.field", 'not docstring'),
+        ("C4.field", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("__doc__", None),])
+    def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
             'docstring'
@@ -903,19 +919,7 @@
         code_w.remove_docstrings(self.space)
         dict_w = self.space.newdict();
         code_w.exec_code(self.space, dict_w, dict_w)
-
-        yield self.check, dict_w, "f1.__doc__", None
-        yield self.check, dict_w, "f2.__doc__", 'docstring'
-        yield self.check, dict_w, "f2()", 'docstring'
-        yield self.check, dict_w, "f3.__doc__", None
-        yield self.check, dict_w, "f3()", 'bar'
-        yield self.check, dict_w, "C1.__doc__", None
-        yield self.check, dict_w, "C2.__doc__", 'docstring'
-        yield self.check, dict_w, "C3.field", 'not docstring'
-        yield self.check, dict_w, "C4.field", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "__doc__", None
+        self.check(dict_w, expr, result)
 
     def test_assert_skipping(self):
         space = self.space
@@ -1111,7 +1115,7 @@
             return d['f'](5)
         """)
         assert 'generator' in space.str_w(space.repr(w_generator))
-        
+
     def test_list_comprehension(self):
         source = "def f(): [i for i in l]"
         source2 = "def f(): [i for i in l for j in l]"
@@ -1152,3 +1156,22 @@
             counts = self.count_instructions(source)
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
+
+    def test_dont_fold_huge_powers(self):
+        for source in (
+            "2 ** 3000",         # not constant-folded: too big
+            "(-2) ** 3000",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER in counts
+
+        for source in (
+            "2 ** 2000",         # constant-folded
+            "2 ** -3000",
+            "1.001 ** 3000",
+            "1 ** 3000.0",
+            ):
+            source = 'def f(): %s' % source
+            counts = self.count_instructions(source)
+            assert ops.BINARY_POWER not in counts

From pypy.commits at gmail.com  Wed Aug 24 19:27:25 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 24 Aug 2016 16:27:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: tweak docs
Message-ID: <57be2d5d.436ec20a.d0586.b3b0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86518:3fefa85e84f5
Date: 2016-08-25 09:25 +1000
http://bitbucket.org/pypy/pypy/changeset/3fefa85e84f5/

Log:	tweak docs

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -135,7 +135,7 @@
     RPython functions, eventually exhausting the stack, while at
     app-level the traceback is very short
 
-.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
 
 Please update, and continue to help us make PyPy better.
 
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.4.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 5.3+
-==========================
+=========================
+What's new in PyPy2.7 5.4
+=========================
 
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1

From pypy.commits at gmail.com  Wed Aug 24 19:07:17 2016
From: pypy.commits at gmail.com (vext01)
Date: Wed, 24 Aug 2016 16:07:17 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Fix a load of W^X
 violations touched in the x86 backend tests.
Message-ID: <57be28a5.d42f1c0a.d0cb9.e07c@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86511:c7822af728c9
Date: 2016-08-24 23:48 +0100
http://bitbucket.org/pypy/pypy/changeset/c7822af728c9/

Log:	Fix a load of W^X violations touched in the x86 backend tests.

diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -230,8 +230,11 @@
         targetindex = self._baserelpos
         while targetindex >= 0:
             dst = rffi.cast(rffi.CCHARP, addr + targetindex)
+            # XXX see if we can move the page fiddling out the loop
+            rmmap.set_pages_writable(dst, blocksize)
             for j in range(blocksize):
                 dst[j] = block.data[j]
+            rmmap.set_pages_executable(dst, blocksize)
             block = block.prev
             blocksize = self.SUBBLOCK_SIZE
             targetindex -= self.SUBBLOCK_SIZE
diff --git a/rpython/jit/backend/llsupport/gcreftracer.py b/rpython/jit/backend/llsupport/gcreftracer.py
--- a/rpython/jit/backend/llsupport/gcreftracer.py
+++ b/rpython/jit/backend/llsupport/gcreftracer.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.jit.backend.llsupport.symbolic import WORD
+from rpython.rlib.rmmap import set_pages_executable, set_pages_writable
 
 
 GCREFTRACER = lltype.GcStruct(
@@ -43,7 +44,10 @@
 def make_boehm_tracer(array_base_addr, gcrefs):
     # copy the addresses, but return 'gcrefs' as the object that must be
     # kept alive
-    for i in range(len(gcrefs)):
+    n_gcrefs = len(gcrefs)
+    set_pages_writable(array_base_addr, n_gcrefs * WORD)
+    for i in range(n_gcrefs):
         p = rffi.cast(rffi.SIGNEDP, array_base_addr + i * WORD)
         p[0] = rffi.cast(lltype.Signed, gcrefs[i])
+    set_pages_executable(array_base_addr, n_gcrefs * WORD)
     return gcrefs
diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py
--- a/rpython/jit/backend/x86/codebuf.py
+++ b/rpython/jit/backend/x86/codebuf.py
@@ -7,6 +7,7 @@
 from rpython.jit.backend.x86.regloc import LocationCodeBuilder
 from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64, WORD
 from rpython.jit.backend.x86 import valgrind
+from rpython.rlib.rmmap import  set_pages_writable, set_pages_executable
 
 # XXX: Seems nasty to change the superclass of MachineCodeBlockWrapper
 # like this
@@ -51,5 +52,6 @@
                 p = addr + reloc
                 adr = rffi.cast(rffi.INTP, p - 4)
                 adr[0] = rffi.cast(rffi.INT, intmask(adr[0]) - p)
+
         valgrind.discard_translations(addr, self.get_relative_pos())
         self._dump(addr, "jit-backend-dump", backend_name)
diff --git a/rpython/jit/backend/x86/detect_feature.py b/rpython/jit/backend/x86/detect_feature.py
--- a/rpython/jit/backend/x86/detect_feature.py
+++ b/rpython/jit/backend/x86/detect_feature.py
@@ -1,17 +1,23 @@
 import sys
 import struct
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rmmap import alloc, free
+from rpython.rlib.rmmap import (alloc, free, set_pages_writable,
+                                set_pages_executable)
+
+CPUINFO_ALLOC_SZ = 4096
+
 
 def cpu_info(instr):
-    data = alloc(4096)
+    data = alloc(CPUINFO_ALLOC_SZ)
     pos = 0
+    set_pages_writable(data, CPUINFO_ALLOC_SZ)
     for c in instr:
         data[pos] = c
         pos += 1
+    set_pages_executable(data, CPUINFO_ALLOC_SZ)
     fnptr = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Signed)), data)
     code = fnptr()
-    free(data, 4096)
+    free(data, CPUINFO_ALLOC_SZ)
     return code
 
 def detect_sse2():
diff --git a/rpython/jit/backend/x86/test/test_regloc.py b/rpython/jit/backend/x86/test/test_regloc.py
--- a/rpython/jit/backend/x86/test/test_regloc.py
+++ b/rpython/jit/backend/x86/test/test_regloc.py
@@ -7,6 +7,7 @@
 from rpython.jit.backend.x86 import codebuf
 from rpython.jit.backend.x86.callbuilder import follow_jump
 from rpython.rlib.rarithmetic import intmask
+from rpython.rlib import rmmap
 import py.test
 
 class LocationCodeBuilder32(CodeBuilder32, LocationCodeBuilder):
@@ -78,7 +79,7 @@
         mc = codebuf.MachineCodeBlockWrapper()
         mc.CALL(ImmedLoc(target))
         length = mc.get_relative_pos()
-        buf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
+        buf = rmmap.alloc(length)
         rawstart = rffi.cast(lltype.Signed, buf)
         if IS_X86_32:
             assert length == 5
@@ -103,17 +104,18 @@
                     "\x41\xFF\xD3")                 # CALL *%r11
         mc.copy_to_raw_memory(rawstart)
         assert ''.join([buf[i] for i in range(length)]) == expected
-        lltype.free(buf, flavor='raw')
+        rmmap.free(buf, length)
 
 class Fake32CodeBlockWrapper(codebuf.MachineCodeBlockWrapper):
     def check_stack_size_at_ret(self):
         pass
         
 def test_follow_jump_instructions_32():
-    buf = lltype.malloc(rffi.CCHARP.TO, 80, flavor='raw')
+    size = 80
+    buf = rmmap.alloc(size)
     raw = rffi.cast(lltype.Signed, buf)
     if not fits_in_32bits(raw):
-        lltype.free(buf, flavor='raw')
+        rmmap.free(buf, size)
         py.test.skip("not testable")
     mc = Fake32CodeBlockWrapper(); mc.WORD = 4; mc.relocations = []
     mc.RET()
@@ -137,7 +139,7 @@
     assert buf[43] == '\xFF'
     assert buf[44] == '\xFF'
     assert follow_jump(raw + 40) == raw
-    lltype.free(buf, flavor='raw')
+    rmmap.free(buf)
 
 
 class Test64Bits:

From pypy.commits at gmail.com  Wed Aug 24 19:27:23 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 24 Aug 2016 16:27:23 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: update version to 5.4.0
Message-ID: <57be2d5b.e97ac20a.936fc.b080@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86517:a1c30344b9b9
Date: 2016-08-25 09:10 +1000
http://bitbucket.org/pypy/pypy/changeset/a1c30344b9b9/

Log:	update version to 5.4.0

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.4.1-alpha0"
-#define PYPY_VERSION_NUM  0x05040100
+#define PYPY_VERSION "5.4.0"
+#define PYPY_VERSION_NUM  0x05040000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 4, 0, "final", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Thu Aug 25 01:36:56 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 24 Aug 2016 22:36:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: revert last commit,
 remove already defined function in RPython (is provided in C)
Message-ID: <57be83f8.17a61c0a.dead8.2f10@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86519:2a28c3d0f49d
Date: 2016-08-25 07:34 +0200
http://bitbucket.org/pypy/pypy/changeset/2a28c3d0f49d/

Log:	revert last commit, remove already defined function in RPython (is
	provided in C)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -516,7 +516,7 @@
     'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs',
     '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT',
 
-    'PyBuffer_Release',
+    'PyObject_GetBuffer', 'PyBuffer_Release',
 
     'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr',
     'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr',
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -5,33 +5,8 @@
     cpython_api, CANNOT_FAIL, Py_buffer)
 from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
 
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, w_obj):
-    """Return 1 if obj supports the buffer interface otherwise 0."""
-    return 0  # the bf_getbuffer field is never filled by cpyext
-
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
-             rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error."""
-    raise oefmt(space.w_TypeError,
-                "PyPy does not yet implement the new buffer interface")
+# PyObject_GetBuffer has been removed, it is defined in abstract.c
+# PyObject_CheckBuffer is also already defined
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
 def PyBuffer_IsContiguous(space, view, fortran):

From pypy.commits at gmail.com  Thu Aug 25 03:22:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 25 Aug 2016 00:22:57 -0700 (PDT)
Subject: [pypy-commit] buildbot default: Forbid "stop build" to be clicked
 with a blank field. More precisely, a
Message-ID: <57be9cd1.d8011c0a.577cb.5d9e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1015:1767edfa9b72
Date: 2016-08-25 09:22 +0200
http://bitbucket.org/pypy/buildbot/changeset/1767edfa9b72/

Log:	Forbid "stop build" to be clicked with a blank field. More
	precisely, a reason must be given which starts with "!". This
	condition is written down clearly in the error we get if we don't
	specify it, so it should not deter human beings, but be enough to
	stop accidents or 3rd-party bots.

diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -6,6 +6,7 @@
 from buildbot.buildslave import BuildSlave
 from buildbot.status.html import WebStatus
 from buildbot.status.web import authz
+from buildbot.process.build import Build
 #from buildbot import manhole
 from pypybuildbot.pypylist import PyPyList, NumpyStatusList
 from pypybuildbot.ircbot import IRC  # side effects
@@ -18,6 +19,14 @@
             raise ValidationError, "Please write your name in the corresponding field."
         return ForceScheduler.force(self, owner, builder_name, **kwargs)
 
+# Forbid "stop build" without a reason that starts with "!"
+def _checkStopBuild(self, reason=""):
+    if ": !" not in reason:
+        raise ValidationError, "Please write a reason that starts with '!'."
+    return _baseStopBuild(self, reason)
+_baseStopBuild = Build.stopBuild
+Build.stopBuild = _checkStopBuild
+
 
 if we_are_debugging():
     channel = '#buildbot-test'

From pypy.commits at gmail.com  Thu Aug 25 03:33:48 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 25 Aug 2016 00:33:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge py3k
Message-ID: <57be9f5c.11331c0a.2d6b2.4a2b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86520:d13f538365f5
Date: 2016-08-25 09:33 +0200
http://bitbucket.org/pypy/pypy/changeset/d13f538365f5/

Log:	merge py3k

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -3,35 +3,10 @@
 from rpython.rlib import buffer
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer)
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
 
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, w_obj):
-    """Return 1 if obj supports the buffer interface otherwise 0."""
-    return 0  # the bf_getbuffer field is never filled by cpyext
-
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
-             rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error."""
-    raise oefmt(space.w_TypeError,
-                "PyPy does not yet implement the new buffer interface")
+# PyObject_GetBuffer has been removed, it is defined in abstract.c
+# PyObject_CheckBuffer is also already defined
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
 def PyBuffer_IsContiguous(space, view, fortran):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -20,6 +20,7 @@
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import widen
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
 from pypy.module.sys.version import CPYTHON_VERSION
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -144,7 +144,7 @@
             # XXX the two lines above take a few seconds to run whenever
             # we initialize the space; for tests, use a simpler version
             from pypy.module.sys.interp_encoding import base_encoding
-            self.filesystemencoding = space.wrap(base_encoding)
+            self.filesystemencoding = base_encoding
 
     def flush_std_files(self, space):
         w_stdout = space.sys.getdictvalue(space, 'stdout')

From pypy.commits at gmail.com  Thu Aug 25 03:35:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 25 Aug 2016 00:35:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merged in
 marky1991/pypy_new/py3k_add_terminal_size (pull request #475)
Message-ID: <57be9fce.e129c20a.4ccf.2438@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3k
Changeset: r86527:80a83063cd45
Date: 2016-08-25 09:34 +0200
http://bitbucket.org/pypy/pypy/changeset/80a83063cd45/

Log:	Merged in marky1991/pypy_new/py3k_add_terminal_size (pull request
	#475)

	Add os.get_terminal_size (py3k)

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -19,6 +19,7 @@
         'statvfs_result': 'app_posix.statvfs_result',
         'uname_result': 'app_posix.uname_result',
         'urandom': 'app_posix.urandom',
+        'terminal_size': 'app_posix.terminal_size'
     }
     if os.name == 'nt':
         del appleveldefs['urandom'] # at interp on win32
@@ -74,6 +75,7 @@
         'abort': 'interp_posix.abort',
         'urandom': 'interp_posix.urandom',
         'device_encoding': 'interp_posix.device_encoding',
+        'get_terminal_size': 'interp_posix.get_terminal_size'
     }
 
     if hasattr(os, 'chown'):
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -92,6 +92,12 @@
     version  = structseqfield(3, "operating system version")
     machine  = structseqfield(4, "hardware identifier")
 
+class terminal_size(metaclass=structseqtype):
+
+    name = "os.terminal_size"
+
+    columns  = structseqfield(0, "width of the terminal window in characters")
+    lines = structseqfield(1, "height of the terminal window in characters")
 
 if osname == 'posix':
     # POSIX: we want to check the file descriptor when fdopen() is called,
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -8,16 +8,18 @@
     # some Pythons don't have errno.ENOTSUP
     ENOTSUP = 0
 
-from rpython.rlib import rposix, rposix_stat
+from rpython.rlib import rposix, rposix_stat, rfile
 from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint
+from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint, r_int
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rtyper.lltypesystem import lltype
 from rpython.tool.sourcetools import func_with_new_name
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, Unwrapper
 from pypy.interpreter.error import (
-    OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror)
+    OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror,
+    exception_from_saved_errno)
 from pypy.interpreter.executioncontext import ExecutionContext
 
 
@@ -2142,3 +2144,50 @@
         have_functions.append("HAVE_%s" % name)
 if _WIN32:
     have_functions.append("HAVE_MS_WINDOWS")
+   
+def get_terminal_size(space, w_fd=None):
+    if w_fd is None:
+        fd = rfile.RFile(rfile.c_stdout(), close2=(None, None)).fileno()
+    else:
+        if not space.isinstance_w(w_fd, space.w_int):
+            raise oefmt(space.w_TypeError,
+                        "an integer is required, got %T", w_fd)
+        else:
+            fd = space.c_int_w(w_fd)
+
+    if _WIN32:
+        if fd == 0:
+            handle_id = rwin32.STD_INPUT_HANDLE
+        elif fd == 1:
+            handle_id = rwin32.STD_OUTPUT_HANDLE
+        elif fd == 2:
+            handle_id = rwin32.STD_ERROR_HANDLE
+        else:
+            raise oefmt(space.w_ValueError, "bad file descriptor")
+
+        handle = rwin32.GetStdHandle(handle_id)
+
+        if handle == rwin32.NULL_HANDLE:
+            raise oefmt(space.w_OSError, "handle cannot be retrieved")
+        elif handle == rwin32.INVALID_HANDLE_VALUE:
+            raise rwin32.lastSavedWindowsError()
+        with lltype.scoped_alloc(rwin32.CONSOLE_SCREEN_BUFFER_INFO) as buffer_info: 
+            success = rwin32.GetConsoleScreenBufferInfo(handle, buffer_info)
+            if not success:
+                raise rwin32.lastSavedWindowsError()
+            w_columns = space.wrap(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
+            w_lines = space.wrap(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
+    else:
+        with lltype.scoped_alloc(rposix.WINSIZE) as winsize: 
+            failed = rposix.c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
+            if failed:
+                raise exception_from_saved_errno(space, space.w_OSError)
+
+            w_columns = space.wrap(r_uint(winsize.c_ws_col))
+            w_lines = space.wrap(r_uint(winsize.c_ws_row))
+
+    w_tuple = space.newtuple([w_columns, w_lines])
+    w_terminal_size = space.getattr(space.getbuiltinmodule(os.name),
+                                    space.wrap('terminal_size'))
+
+    return space.call_function(w_terminal_size, w_tuple)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -258,6 +258,7 @@
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
+        TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),
@@ -265,6 +266,12 @@
                            ('tms_cutime', rffi.INT),
                            ('tms_cstime', rffi.INT)])
 
+        WINSIZE = rffi_platform.Struct(
+            'struct winsize', [('ws_row', rffi.USHORT),
+                           ('ws_col', rffi.USHORT),
+                           ('ws_xpixel', rffi.USHORT),
+                           ('ws_ypixel', rffi.USHORT)])
+
     GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
     SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")
 
@@ -628,6 +635,8 @@
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
     c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+    c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 else:
     dirent_config = {}
 
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -59,6 +59,24 @@
         SYSTEMTIME = rffi_platform.Struct('SYSTEMTIME',
                                           [])
 
+        Struct = rffi_platform.Struct
+        COORD = Struct("COORD",
+                       [("X", rffi.SHORT),
+                        ("Y", rffi.SHORT)])
+
+        SMALL_RECT = Struct("SMALL_RECT",
+                            [("Left", rffi.SHORT),
+                             ("Top", rffi.SHORT),
+                             ("Right", rffi.SHORT),
+                             ("Bottom", rffi.SHORT)])
+
+        CONSOLE_SCREEN_BUFFER_INFO = Struct("CONSOLE_SCREEN_BUFFER_INFO",
+                                            [("dwSize", COORD),
+                                             ("dwCursorPosition", COORD),
+                                             ("wAttributes", WORD.ctype_hint),
+                                             ("srWindow", SMALL_RECT),
+                                             ("dwMaximumWindowSize", COORD)])
+
         OSVERSIONINFOEX = rffi_platform.Struct(
             'OSVERSIONINFOEX',
             [('dwOSVersionInfoSize', rffi.UINT),
@@ -93,7 +111,8 @@
                        PROCESS_VM_WRITE
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
-                       WC_NO_BEST_FIT_CHARS
+                       WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
+                       STD_ERROR_HANDLE
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -444,3 +463,13 @@
         return rffi.cast(lltype.Signed, _GetConsoleOutputCP())
 
     _wenviron_items, _wgetenv, _wputenv = make_env_impls(win32=True)
+
+
+    _GetStdHandle = winexternal(
+        'GetStdHandle', [DWORD], HANDLE)
+
+    def GetStdHandle(handle_id):
+        return _GetStdHandle(handle_id)
+    CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
+    GetConsoleScreenBufferInfo = winexternal(
+        "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)

From pypy.commits at gmail.com  Thu Aug 25 03:36:07 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 25 Aug 2016 00:36:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3k_add_terminal_size: Posting what I have so
 far.
Message-ID: <57be9fe7.448e1c0a.4587.4bca@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k_add_terminal_size
Changeset: r86521:7cf5e409328e
Date: 2016-08-23 19:41 -0400
http://bitbucket.org/pypy/pypy/changeset/7cf5e409328e/

Log:	Posting what I have so far.

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -74,6 +74,7 @@
         'abort': 'interp_posix.abort',
         'urandom': 'interp_posix.urandom',
         'device_encoding': 'interp_posix.device_encoding',
+        'get_terminal_size': 'interp_posix.get_terminal_size'
     }
 
     if hasattr(os, 'chown'):
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2142,3 +2142,49 @@
         have_functions.append("HAVE_%s" % name)
 if _WIN32:
     have_functions.append("HAVE_MS_WINDOWS")
+   
+def get_terminal_size(space, w_fd=None):
+    if w_fd is None:
+        w_stdout = space.sys.getdictvalue(space, 'stdout')
+        fd = space.int_w(w_stdout.fileno_w(space))
+    else:
+        if not space.isinstance_w(w_fd, space.w_int):
+            raise oefmt(space.w_TypeError,
+                        "an integer is required, got %T", w_fd)
+        else:
+            fd = space.c_int_w(w_fd)
+
+    if _WIN32:
+        if fd == 0:
+            handle_id = rwin32.STD_INPUT_HANDLE
+        elif fd == 1:
+            handle_id = rwin32.STD_OUTPUT_HANDLE
+        elif fd == 2:
+            handle_id = rwin32.STD_ERROR_HANDLE
+        else:
+            raise oefmt(space.w_ValueError, "bad file descriptor")
+
+        handle = rwin32.GetStdHandle(handle_id)
+
+        if handle == rwin32.NULL_HANDLE:
+            raise oefmt(OSError, "handle cannot be retrieved")
+        elif handle == rwin32.INVALID_HANDLE_VALUE:
+            raise rwin32.lastSavedWindowsError()
+        with lltype.scoped_alloc(CONSOLE_SCREEN_BUFFER_INFO) as buffer_info: 
+            success = GetConsoleScreenBufferInfo(handle, buffer_info)
+            if not success:
+                raise rwin32.lastSavedWindowsError()
+            columns = buffer_info.srWindow.Right - buffer_info.srWindow.Left + 1
+            lines = buffer_info.srWindow.Bottom - buffer_info.srWindow.Top + 1
+    else:
+        # Assuming that all supported platforms will have ioctl at least
+        with lltype.scoped_alloc(rposix.WINSIZE) as winsize: 
+            failed = c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
+            if failed:
+                raise exception_from_saved_errno(space, space.w_OSError)
+
+            # TODO: Wrap this into a python_lvel int (somehow)
+            columns = space.wrap(winsize.c_ws_col)
+            lines = space.wrap(winsize.c_ws_row)
+
+    return space.newtuple([columns, lines])
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -258,6 +258,7 @@
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
+        TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),
@@ -265,6 +266,14 @@
                            ('tms_cutime', rffi.INT),
                            ('tms_cstime', rffi.INT)])
 
+        WINSIZE = rffi_platform.Struct(
+            'struct winsize', [('ws_row', rffi.USHORT),
+                           ('ws_col', rffi.USHORT),
+                           ('ws_xpixel', rffi.USHORT),
+                           ('ws_ypixel', rffi.USHORT)])
+         
+
+
     GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
     SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")
 
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -59,6 +59,24 @@
         SYSTEMTIME = rffi_platform.Struct('SYSTEMTIME',
                                           [])
 
+        Struct = rffi_platform.Struct
+        COORD = Struct("COORD",
+                       [("X", rffi.SHORT),
+                        ("Y", rffi.SHORT)])
+
+        SMALL_RECT = Struct("SMALL_RECT",
+                            [("Left", rffi.SHORT),
+                             ("Top", rffi.SHORT),
+                             ("Right", rffi.SHORT),
+                             ("Bottom", rffi.SHORT)])
+
+        CONSOLE_SCREEN_BUFFER_INFO = Struct("CONSOLE_SCREEN_BUFFER_INFO",
+                                            [("dwSize", COORD),
+                                             ("dwCursorPosition", COORD),
+                                             ("wAttributes", WORD),
+                                             ("srWindow", SMALL_RECT),
+                                             ("dwMaximumWindowSize", COORD)])
+
         OSVERSIONINFOEX = rffi_platform.Struct(
             'OSVERSIONINFOEX',
             [('dwOSVersionInfoSize', rffi.UINT),
@@ -93,7 +111,8 @@
                        PROCESS_VM_WRITE
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
-                       WC_NO_BEST_FIT_CHARS
+                       WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
+                       STD_ERROR_HANDLE
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -444,3 +463,13 @@
         return rffi.cast(lltype.Signed, _GetConsoleOutputCP())
 
     _wenviron_items, _wgetenv, _wputenv = make_env_impls(win32=True)
+
+
+    _GetStdHandle = winexternal(
+        'GetStdHandle', [DWORD], HANDLE)
+
+    def GetStdHandle(handle_id):
+        return _GetStdHandle(handle_id)
+
+    _GetConsoleScreenBufferInfo = winexternal(
+        "GetConsoleScreenBufferInfo", [HANDLE, csbi], BOOL)

From pypy.commits at gmail.com  Thu Aug 25 03:36:08 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 25 Aug 2016 00:36:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3k_add_terminal_size: lib-python test passes
 for linux now at least.
Message-ID: <57be9fe8.469d1c0a.301f7.54b5@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k_add_terminal_size
Changeset: r86522:8befc19a63b8
Date: 2016-08-24 09:55 -0400
http://bitbucket.org/pypy/pypy/changeset/8befc19a63b8/

Log:	lib-python test passes for linux now at least.

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -19,6 +19,7 @@
         'statvfs_result': 'app_posix.statvfs_result',
         'uname_result': 'app_posix.uname_result',
         'urandom': 'app_posix.urandom',
+        'terminal_size': 'app_posix.terminal_size'
     }
     if os.name == 'nt':
         del appleveldefs['urandom'] # at interp on win32
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -92,6 +92,12 @@
     version  = structseqfield(3, "operating system version")
     machine  = structseqfield(4, "hardware identifier")
 
+class terminal_size(metaclass=structseqtype):
+
+    name = osname + ".terminal_size"
+
+    columns  = structseqfield(0, "width of the terminal window in characters")
+    lines = structseqfield(1, "height of the terminal window in characters")
 
 if osname == 'posix':
     # POSIX: we want to check the file descriptor when fdopen() is called,
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -13,11 +13,13 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rtyper.lltypesystem import lltype
 from rpython.tool.sourcetools import func_with_new_name
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, Unwrapper
 from pypy.interpreter.error import (
-    OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror)
+    OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror,
+    exception_from_saved_errno)
 from pypy.interpreter.executioncontext import ExecutionContext
 
 
@@ -2179,7 +2181,7 @@
     else:
         # Assuming that all supported platforms will have ioctl at least
         with lltype.scoped_alloc(rposix.WINSIZE) as winsize: 
-            failed = c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
+            failed = rposix.c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
             if failed:
                 raise exception_from_saved_errno(space, space.w_OSError)
 
@@ -2187,4 +2189,8 @@
             columns = space.wrap(winsize.c_ws_col)
             lines = space.wrap(winsize.c_ws_row)
 
-    return space.newtuple([columns, lines])
+    w_tuple = space.newtuple([columns, lines])
+    w_terminal_size = space.getattr(space.getbuiltinmodule(os.name),
+                                    space.wrap('terminal_size'))
+
+    return space.call_function(w_terminal_size, w_tuple)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -637,6 +637,8 @@
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
     c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+    c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 else:
     dirent_config = {}
 

From pypy.commits at gmail.com  Thu Aug 25 03:36:11 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 25 Aug 2016 00:36:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3k_add_terminal_size: get_terminal_size
 lib-python tests pass on windows after translation.
Message-ID: <57be9feb.85261c0a.721b3.631f@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k_add_terminal_size
Changeset: r86523:8d4194d34b83
Date: 2016-08-24 13:37 -0400
http://bitbucket.org/pypy/pypy/changeset/8d4194d34b83/

Log:	get_terminal_size lib-python tests pass on windows after
	translation.

diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -94,7 +94,7 @@
 
 class terminal_size(metaclass=structseqtype):
 
-    name = osname + ".terminal_size"
+    name = "os.terminal_size"
 
     columns  = structseqfield(0, "width of the terminal window in characters")
     lines = structseqfield(1, "height of the terminal window in characters")
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -11,7 +11,7 @@
 from rpython.rlib import rposix, rposix_stat
 from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint
+from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint, r_int
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.lltypesystem import lltype
 from rpython.tool.sourcetools import func_with_new_name
@@ -2169,15 +2169,16 @@
         handle = rwin32.GetStdHandle(handle_id)
 
         if handle == rwin32.NULL_HANDLE:
-            raise oefmt(OSError, "handle cannot be retrieved")
+            raise oefmt(space.w_OSError, "handle cannot be retrieved")
         elif handle == rwin32.INVALID_HANDLE_VALUE:
             raise rwin32.lastSavedWindowsError()
-        with lltype.scoped_alloc(CONSOLE_SCREEN_BUFFER_INFO) as buffer_info: 
-            success = GetConsoleScreenBufferInfo(handle, buffer_info)
+        with lltype.scoped_alloc(rwin32.CONSOLE_SCREEN_BUFFER_INFO) as buffer_info: 
+            success = rwin32.GetConsoleScreenBufferInfo(handle, buffer_info)
             if not success:
                 raise rwin32.lastSavedWindowsError()
-            columns = buffer_info.srWindow.Right - buffer_info.srWindow.Left + 1
-            lines = buffer_info.srWindow.Bottom - buffer_info.srWindow.Top + 1
+            # TODO: Is the typing here right?
+            w_columns = space.wrap(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
+            w_lines = space.wrap(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
     else:
         # Assuming that all supported platforms will have ioctl at least
         with lltype.scoped_alloc(rposix.WINSIZE) as winsize: 
@@ -2186,10 +2187,11 @@
                 raise exception_from_saved_errno(space, space.w_OSError)
 
             # TODO: Wrap this into a python_lvel int (somehow)
-            columns = space.wrap(winsize.c_ws_col)
-            lines = space.wrap(winsize.c_ws_row)
+            # TODO: is this right?
+            w_columns = space.wrap(r_uint(winsize.c_ws_col))
+            w_lines = space.wrap(r_uint(winsize.c_ws_row))
 
-    w_tuple = space.newtuple([columns, lines])
+    w_tuple = space.newtuple([w_columns, w_lines])
     w_terminal_size = space.getattr(space.getbuiltinmodule(os.name),
                                     space.wrap('terminal_size'))
 
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -73,7 +73,7 @@
         CONSOLE_SCREEN_BUFFER_INFO = Struct("CONSOLE_SCREEN_BUFFER_INFO",
                                             [("dwSize", COORD),
                                              ("dwCursorPosition", COORD),
-                                             ("wAttributes", WORD),
+                                             ("wAttributes", WORD.ctype_hint),
                                              ("srWindow", SMALL_RECT),
                                              ("dwMaximumWindowSize", COORD)])
 
@@ -470,6 +470,6 @@
 
     def GetStdHandle(handle_id):
         return _GetStdHandle(handle_id)
-
-    _GetConsoleScreenBufferInfo = winexternal(
-        "GetConsoleScreenBufferInfo", [HANDLE, csbi], BOOL)
+    CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
+    GetConsoleScreenBufferInfo = winexternal(
+        "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)

From pypy.commits at gmail.com  Thu Aug 25 03:36:13 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 25 Aug 2016 00:36:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3k_add_terminal_size: Slight tweaks
Message-ID: <57be9fed.02c41c0a.78b42.9741@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k_add_terminal_size
Changeset: r86524:007aa1dff8cb
Date: 2016-08-24 21:34 -0400
http://bitbucket.org/pypy/pypy/changeset/007aa1dff8cb/

Log:	Slight tweaks

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -8,7 +8,7 @@
     # some Pythons don't have errno.ENOTSUP
     ENOTSUP = 0
 
-from rpython.rlib import rposix, rposix_stat
+from rpython.rlib import rposix, rposix_stat, rfile
 from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint, r_int
@@ -2147,8 +2147,7 @@
    
 def get_terminal_size(space, w_fd=None):
     if w_fd is None:
-        w_stdout = space.sys.getdictvalue(space, 'stdout')
-        fd = space.int_w(w_stdout.fileno_w(space))
+        fd = rfile.c_fileno(rfile.c_stdout)
     else:
         if not space.isinstance_w(w_fd, space.w_int):
             raise oefmt(space.w_TypeError,
@@ -2176,18 +2175,14 @@
             success = rwin32.GetConsoleScreenBufferInfo(handle, buffer_info)
             if not success:
                 raise rwin32.lastSavedWindowsError()
-            # TODO: Is the typing here right?
             w_columns = space.wrap(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
             w_lines = space.wrap(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
     else:
-        # Assuming that all supported platforms will have ioctl at least
         with lltype.scoped_alloc(rposix.WINSIZE) as winsize: 
             failed = rposix.c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
             if failed:
                 raise exception_from_saved_errno(space, space.w_OSError)
 
-            # TODO: Wrap this into a python_lvel int (somehow)
-            # TODO: is this right?
             w_columns = space.wrap(r_uint(winsize.c_ws_col))
             w_lines = space.wrap(r_uint(winsize.c_ws_row))
 

From pypy.commits at gmail.com  Thu Aug 25 03:36:15 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 25 Aug 2016 00:36:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3k_add_terminal_size: Whitespace fix.
Message-ID: <57be9fef.82cbc20a.a3db7.1dd2@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k_add_terminal_size
Changeset: r86525:090a7002785e
Date: 2016-08-24 21:37 -0400
http://bitbucket.org/pypy/pypy/changeset/090a7002785e/

Log:	Whitespace fix.

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -271,8 +271,6 @@
                            ('ws_col', rffi.USHORT),
                            ('ws_xpixel', rffi.USHORT),
                            ('ws_ypixel', rffi.USHORT)])
-         
-
 
     GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
     SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")

From pypy.commits at gmail.com  Thu Aug 25 03:36:16 2016
From: pypy.commits at gmail.com (marky1991)
Date: Thu, 25 Aug 2016 00:36:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3k_add_terminal_size: Fix translation.
Message-ID: <57be9ff0.c310c20a.bb491.27a2@mx.google.com>

Author: Mark Young <marky1991 at gmail.com>
Branch: py3k_add_terminal_size
Changeset: r86526:8af26ef17f86
Date: 2016-08-24 22:42 -0400
http://bitbucket.org/pypy/pypy/changeset/8af26ef17f86/

Log:	Fix translation.

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2147,7 +2147,7 @@
    
 def get_terminal_size(space, w_fd=None):
     if w_fd is None:
-        fd = rfile.c_fileno(rfile.c_stdout)
+        fd = rfile.RFile(rfile.c_stdout(), close2=(None, None)).fileno()
     else:
         if not space.isinstance_w(w_fd, space.w_int):
             raise oefmt(space.w_TypeError,

From pypy.commits at gmail.com  Thu Aug 25 05:36:22 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 25 Aug 2016 02:36:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: merge py3.5
Message-ID: <57bebc16.058a1c0a.a3491.7afa@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86528:225353ef1b51
Date: 2016-08-25 09:35 +0200
http://bitbucket.org/pypy/pypy/changeset/225353ef1b51/

Log:	merge py3.5

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -3,35 +3,10 @@
 from rpython.rlib import buffer
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer)
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
 
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, w_obj):
-    """Return 1 if obj supports the buffer interface otherwise 0."""
-    return 0  # the bf_getbuffer field is never filled by cpyext
-
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
-             rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error."""
-    raise oefmt(space.w_TypeError,
-                "PyPy does not yet implement the new buffer interface")
+# PyObject_GetBuffer has been removed, it is defined in abstract.c
+# PyObject_CheckBuffer is also already defined
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
 def PyBuffer_IsContiguous(space, view, fortran):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -20,6 +20,7 @@
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import widen
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
 from pypy.module.sys.version import CPYTHON_VERSION
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -144,7 +144,7 @@
             # XXX the two lines above take a few seconds to run whenever
             # we initialize the space; for tests, use a simpler version
             from pypy.module.sys.interp_encoding import base_encoding
-            self.filesystemencoding = space.wrap(base_encoding)
+            self.filesystemencoding = base_encoding
 
     def flush_std_files(self, space):
         w_stdout = space.sys.getdictvalue(space, 'stdout')

From pypy.commits at gmail.com  Thu Aug 25 05:36:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 25 Aug 2016 02:36:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: yes,
 the first failing test for memoryview tuple indexing
Message-ID: <57bebc18.274fc20a.2eca5.549b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86529:3ba72a821c80
Date: 2016-08-25 11:35 +0200
http://bitbucket.org/pypy/pypy/changeset/3ba72a821c80/

Log:	yes, the first failing test for memoryview tuple indexing

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -1048,6 +1048,7 @@
             return dtype
         if name[0] in 'VSUca' or name[0] in '<>=|' and name[1] in 'VSUca':
             return variable_dtype(space, name)
+        import pdb; pdb.set_trace()
         raise oefmt(space.w_TypeError, 'data type "%s" not understood', name)
     elif space.isinstance_w(w_dtype, space.w_list):
         return _set_metadata_and_copy( space, w_metadata,
diff --git a/pypy/module/micronumpy/test/__init__.py b/pypy/module/micronumpy/test/__init__.py
--- a/pypy/module/micronumpy/test/__init__.py
+++ b/pypy/module/micronumpy/test/__init__.py
@@ -1,2 +1,2 @@
 import py
-py.test.py3k_skip('not yet supported')
+# XXX py.test.py3k_skip('not yet supported')
diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -20,7 +20,7 @@
 for t in types:
     globals()[t] = dtype(t).type
 
-types = ['bool', 'int', 'float', 'complex', 'str', 'string', 'unicode', 'object']
+types = ['bool', 'int', 'float', 'complex', 'str', 'object']
 for t in types:
     globals()[t + '_'] = dtype(t).type
 del types
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -9,7 +9,7 @@
 
     @classmethod
     def setup_class(cls):
-        py.test.py3k_skip("micronumpy not supported on py3k")
+        # XXX py.test.py3k_skip("micronumpy not supported on py3k")
         if option.runappdirect:
             import sys
             if '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -166,7 +166,12 @@
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
 
-    @py.test.skip("needs numpy ndarray")
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestMemoryViewMicroNumPyPy(BaseNumpyAppTest):
+    spaceconfig = dict(usemodules=['array', 'micronumpy'])
+
     def test_tuple_indexing(self):
+        from numpy import ndarray
         content = ndarray(list(range(12)))
         assert memoryview(content)[0,0] == 0

From pypy.commits at gmail.com  Thu Aug 25 06:22:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 25 Aug 2016 03:22:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: Skip if we can't import hypothesis,
 e.g. because we're running with pypy and -A
Message-ID: <57bec6e1.17a61c0a.dead8.9766@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86530:3d655320ad2a
Date: 2016-08-25 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/3d655320ad2a/

Log:	Skip if we can't import hypothesis, e.g. because we're running with
	pypy and -A

diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -1,7 +1,10 @@
 import pytest
 import sys
-from hypothesis import given, strategies, settings
 from pypy.tool.pytest.objspace import gettestobjspace
+try:
+    from hypothesis import given, strategies, settings
+except ImportError:
+    pytest.skip("requires hypothesis")
 
 base_initargs = strategies.sampled_from([
     ("object", (), False),

From pypy.commits at gmail.com  Thu Aug 25 06:23:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 25 Aug 2016 03:23:15 -0700 (PDT)
Subject: [pypy-commit] pypy default: detabbify
Message-ID: <57bec713.c70a1c0a.a52d2.8dee@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86531:3d4985d7873d
Date: 2016-08-25 12:22 +0200
http://bitbucket.org/pypy/pypy/changeset/3d4985d7873d/

Log:	detabbify

diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -11,7 +11,7 @@
 
 class FakeCallAssemblerLoopToken(AbstractDescr):
     def __init__(self, target):
-	self._ll_function_addr = target
+        self._ll_function_addr = target
 
     def repr_of_descr(self):
         return 'looptoken'

From pypy.commits at gmail.com  Thu Aug 25 06:39:14 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 25 Aug 2016 03:39:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: progress on implementing tuple
 indexing (almost done with the logic)
Message-ID: <57becad2.109a1c0a.f5265.a477@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86532:c771970fb98a
Date: 2016-08-25 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/c771970fb98a/

Log:	progress on implementing tuple indexing (almost done with the logic)

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -1048,7 +1048,6 @@
             return dtype
         if name[0] in 'VSUca' or name[0] in '<>=|' and name[1] in 'VSUca':
             return variable_dtype(space, name)
-        import pdb; pdb.set_trace()
         raise oefmt(space.w_TypeError, 'data type "%s" not understood', name)
     elif space.isinstance_w(w_dtype, space.w_list):
         return _set_metadata_and_copy( space, w_metadata,
diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -20,6 +20,8 @@
 for t in types:
     globals()[t] = dtype(t).type
 
+# removed 'string' and 'unicode' from that list to handle an error in
+# make_new_dtype (micronumpy/descriptor.py)
 types = ['bool', 'int', 'float', 'complex', 'str', 'object']
 for t in types:
     globals()[t + '_'] = dtype(t).type
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -79,9 +79,51 @@
         fmtiter.interpret(self.format * self.getlength())
         return space.newlist(fmtiter.result_w)
 
+    def _start_from_tuple(self, space, w_tuple):
+        start = 0
+
+        view = self.buf
+        length = space.len(w_tuple)
+        for i, w_obj in enumerate(w_tuple.getitems_unroll()):
+            value = w_obj.int_w(space)
+            start = self.lookup_dimension(start, dim, index)
+        return start
+
+    def lookup_dimension(self, space, start, dim, index):
+        return start
+
+    def _getitem_tuple_indexed(self, w_index):
+        view = self.buf
+
+        fmt = view.getformat() # TODO adjust format?
+
+        length = space.len_w(w_index)
+        ndim = view.getndim()
+        if length < ndim:
+            raise OperationError(self.w_NotImplementedError, \
+                    self.wrap("sub-views are not implemented"))
+
+        if length > ndim:
+            raise oefmt(self.w_NotImplementedError, \
+                    "cannot index %d-dimension view with %d-element tuple",
+                    length, ndim)
+
+        start = self._start_from_tuple(space, w_index)
+
+        buf = SubBuffer(self.buf, start, self.itemsize)
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(fmt)
+        return fmtiter.result_w[0]
+
+
     def descr_getitem(self, space, w_index):
         self._check_released(space)
-        start, stop, step, size = space.decode_index4(w_index, self.getlength())
+
+        if self.isinstance_w(w_index, self.w_tuple):
+            return self._getitem_tuple_indexed(space, w_index)
+
+        start, stop, step, size = space.decode_index4_or_tuple_index(w_index, \
+                                                              self.getlength())
         itemsize = self.buf.getitemsize()
         if itemsize > 1:
             start *= itemsize
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -173,5 +173,5 @@
 
     def test_tuple_indexing(self):
         from numpy import ndarray
-        content = ndarray(list(range(12)))
+        content = ndarray(list(range(12))).reshape(3,4)
         assert memoryview(content)[0,0] == 0

From pypy.commits at gmail.com  Thu Aug 25 08:22:16 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 25 Aug 2016 05:22:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: finish review of all commits to default
 since last release
Message-ID: <57bee2f8.4152c20a.199e7.957f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86533:fa64ce678bb5
Date: 2016-08-25 22:17 +1000
http://bitbucket.org/pypy/pypy/changeset/fa64ce678bb5/

Log:	finish review of all commits to default since last release

diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -5,8 +5,14 @@
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
 This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
-and fixed many issues and bugs raised by the growing community of PyPy
-users.
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. 
 
 XXXXX MORE ???
 
@@ -25,6 +31,7 @@
 
 .. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
 .. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
 .. _`PyPy`: http://doc.pypy.org
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
@@ -74,13 +81,47 @@
     `issubclass()` as well as `type.__instancecheck__()` and
     `type.__subclasscheck__()`
 
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Support for memoryview attributes (format, itemsize, ...) which also
+    adds support for `PyMemoryView_FromObject`
+
 * Bug Fixes
 
   * Reject `mkdir()` in read-only sandbox filesystems
 
   * Add include guards to pymem.h to enable c++ compilation
 
-  * Fix OpenBSD build breakage and support OpenBSD in VMProf.
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
 
   * Fix for `bytearray('').replace('a', 'ab')` for empty strings
 
@@ -104,10 +145,30 @@
     `MADV_DONTNEED` on freed arenas to release memory back to the
     OS for resource monitoring
 
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
   * Issues reported with our previous release were resolved_ after
     reports from users on our issue tracker at
     https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
 
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
@@ -135,6 +196,22 @@
     RPython functions, eventually exhausting the stack, while at
     app-level the traceback is very short
 
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
 .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
 
 Please update, and continue to help us make PyPy better.

From pypy.commits at gmail.com  Thu Aug 25 08:22:20 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 25 Aug 2016 05:22:20 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: merge default into release
Message-ID: <57bee2fc.d42f1c0a.d0cb9.c880@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86535:aa2985876f22
Date: 2016-08-25 22:21 +1000
http://bitbucket.org/pypy/pypy/changeset/aa2985876f22/

Log:	merge default into release

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -5,8 +5,14 @@
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
 This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
-and fixed many issues and bugs raised by the growing community of PyPy
-users.
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. 
 
 XXXXX MORE ???
 
@@ -25,6 +31,7 @@
 
 .. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
 .. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
 .. _`PyPy`: http://doc.pypy.org
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
@@ -74,13 +81,47 @@
     `issubclass()` as well as `type.__instancecheck__()` and
     `type.__subclasscheck__()`
 
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Support for memoryview attributes (format, itemsize, ...) which also
+    adds support for `PyMemoryView_FromObject`
+
 * Bug Fixes
 
   * Reject `mkdir()` in read-only sandbox filesystems
 
   * Add include guards to pymem.h to enable c++ compilation
 
-  * Fix OpenBSD build breakage and support OpenBSD in VMProf.
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
 
   * Fix for `bytearray('').replace('a', 'ab')` for empty strings
 
@@ -104,10 +145,30 @@
     `MADV_DONTNEED` on freed arenas to release memory back to the
     OS for resource monitoring
 
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
   * Issues reported with our previous release were resolved_ after
     reports from users on our issue tracker at
     https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
 
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
@@ -135,7 +196,23 @@
     RPython functions, eventually exhausting the stack, while at
     app-level the traceback is very short
 
-.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
 
 Please update, and continue to help us make PyPy better.
 
diff --git a/pypy/doc/whatnew-head.rst b/pypy/doc/whatsnew-head.rst
rename from pypy/doc/whatnew-head.rst
rename to pypy/doc/whatsnew-head.rst
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.4.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 5.3+
-==========================
+=========================
+What's new in PyPy2.7 5.4
+=========================
 
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -1,7 +1,10 @@
 import pytest
 import sys
-from hypothesis import given, strategies, settings
 from pypy.tool.pytest.objspace import gettestobjspace
+try:
+    from hypothesis import given, strategies, settings
+except ImportError:
+    pytest.skip("requires hypothesis")
 
 base_initargs = strategies.sampled_from([
     ("object", (), False),
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -11,7 +11,7 @@
 
 class FakeCallAssemblerLoopToken(AbstractDescr):
     def __init__(self, target):
-	self._ll_function_addr = target
+        self._ll_function_addr = target
 
     def repr_of_descr(self):
         return 'looptoken'

From pypy.commits at gmail.com  Thu Aug 25 08:22:18 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 25 Aug 2016 05:22:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: rename file correctly
Message-ID: <57bee2fa.0117c20a.c7a53.9fba@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86534:2dc786dd9cfe
Date: 2016-08-25 22:19 +1000
http://bitbucket.org/pypy/pypy/changeset/2dc786dd9cfe/

Log:	rename file correctly

diff --git a/pypy/doc/whatnew-head.rst b/pypy/doc/whatsnew-head.rst
rename from pypy/doc/whatnew-head.rst
rename to pypy/doc/whatsnew-head.rst

From pypy.commits at gmail.com  Thu Aug 25 16:50:18 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 25 Aug 2016 13:50:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Revert the main part of 26dfa2f67afd since
 it breaks too many random things, 
Message-ID: <57bf5a0a.c15e1c0a.d8803.8d18@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86536:ce4d4b3c159f
Date: 2016-08-25 21:49 +0100
http://bitbucket.org/pypy/pypy/changeset/ce4d4b3c159f/

Log:	Revert the main part of 26dfa2f67afd since it breaks too many random
	things, e.g. it caused pypy/module/imp/test/test_import.py::AppTestM
	ultithreadedImp.test_multithreaded_import to hang.

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -139,12 +139,10 @@
         space = self.space
 
         if not space.config.translating:
-            ##from pypy.module.sys.interp_encoding import _getfilesystemencoding
-            ##self.filesystemencoding = _getfilesystemencoding(space)
+            from pypy.module.sys.interp_encoding import _getfilesystemencoding
+            self.filesystemencoding = _getfilesystemencoding(space)
             # XXX the two lines above take a few seconds to run whenever
-            # we initialize the space; for tests, use a simpler version
-            from pypy.module.sys.interp_encoding import base_encoding
-            self.filesystemencoding = base_encoding
+            # we initialize the space
 
     def flush_std_files(self, space):
         w_stdout = space.sys.getdictvalue(space, 'stdout')

From pypy.commits at gmail.com  Thu Aug 25 21:00:49 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 25 Aug 2016 18:00:49 -0700 (PDT)
Subject: [pypy-commit] pypy default: compilation fixes for 32bit, windows
Message-ID: <57bf94c1.d4e01c0a.a8cdb.b9e4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86538:299b70960727
Date: 2016-08-26 03:40 +0300
http://bitbucket.org/pypy/pypy/changeset/299b70960727/

Log:	compilation fixes for 32bit, windows

diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -1,3 +1,6 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
 #include <Python.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -10,7 +13,7 @@
 /* Structure defines a 1-dimensional strided array */
 typedef struct{
     int* arr;
-    long length;
+    Py_ssize_t length;
 } MyArray;
 
 /* initialize the array with integers 0...length */
@@ -61,13 +64,13 @@
 static int
 PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
 {
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
     // init may have already been called
     if (self->arr.arr != NULL) {
         deallocate_MyArray(&self->arr);
     }
 
-    int length = 0;
-    static char *kwlist[] = {"length", NULL};
     if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
         return -1;
 
@@ -103,16 +106,19 @@
 static int
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
 
-  PyMyArray* self = (PyMyArray*)obj;
   view->obj = (PyObject*)self;
   view->buf = (void*)self->arr.arr;
   view->len = self->arr.length * sizeof(int);
@@ -218,7 +224,6 @@
 #ifdef __GNUC__
 extern __attribute__((visibility("default")))
 #else
-extern __declspec(dllexport)
 #endif
 
 PyMODINIT_FUNC

From pypy.commits at gmail.com  Fri Aug 26 01:20:10 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 25 Aug 2016 22:20:10 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-subclass: copy Py_TPFLAGS_CHECKTYPES,
 Py_TPFLAGS_HAVE_INPLACEOPS from base class when inheriting
Message-ID: <57bfd18a.031dc20a.f5e91.c069@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-subclass
Changeset: r86539:69e5dabb1261
Date: 2016-08-26 12:59 +1000
http://bitbucket.org/pypy/pypy/changeset/69e5dabb1261/

Log:	copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS from base
	class when inheriting

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,7 +120,7 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 """.split()
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -18,7 +18,8 @@
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER)
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -386,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -814,8 +817,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 

From pypy.commits at gmail.com  Fri Aug 26 01:20:12 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 25 Aug 2016 22:20:12 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-subclass: close branch for merging
Message-ID: <57bfd18c.e97ac20a.936fc.c1b4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-subclass
Changeset: r86540:ecfa8861c042
Date: 2016-08-26 13:00 +1000
http://bitbucket.org/pypy/pypy/changeset/ecfa8861c042/

Log:	close branch for merging


From pypy.commits at gmail.com  Fri Aug 26 01:20:16 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 25 Aug 2016 22:20:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <57bfd190.09afc20a.8df27.bcd4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86542:522736f816dc
Date: 2016-08-26 13:03 +1000
http://bitbucket.org/pypy/pypy/changeset/522736f816dc/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,6 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 531050b1f410
+.. startrev: 4176c6f63109
 
 
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.4.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -159,3 +159,7 @@
 .. branch: redirect-assembler-jitlog
 
 Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting

From pypy.commits at gmail.com  Fri Aug 26 01:20:17 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 25 Aug 2016 22:20:17 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: merge default into release
Message-ID: <57bfd191.465d1c0a.3dddb.fad8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86543:c32a98778d43
Date: 2016-08-26 15:18 +1000
http://bitbucket.org/pypy/pypy/changeset/c32a98778d43/

Log:	merge default into release

diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -137,6 +137,8 @@
             lib.gdbm_sync(self.__ll_dbm)
 
 def open(filename, flags='r', mode=0666):
+    if isinstance(filename, unicode):
+        filename = filename.encode()
     if flags[0] == 'r':
         iflags = lib.GDBM_READER
     elif flags[0] == 'w':
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,6 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 531050b1f410
+.. startrev: 4176c6f63109
 
 
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.4.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -159,3 +159,7 @@
 .. branch: redirect-assembler-jitlog
 
 Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,7 +120,7 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 """.split()
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -1,3 +1,6 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
 #include <Python.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -10,7 +13,7 @@
 /* Structure defines a 1-dimensional strided array */
 typedef struct{
     int* arr;
-    long length;
+    Py_ssize_t length;
 } MyArray;
 
 /* initialize the array with integers 0...length */
@@ -61,13 +64,13 @@
 static int
 PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
 {
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
     // init may have already been called
     if (self->arr.arr != NULL) {
         deallocate_MyArray(&self->arr);
     }
 
-    int length = 0;
-    static char *kwlist[] = {"length", NULL};
     if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
         return -1;
 
@@ -103,16 +106,19 @@
 static int
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
 
-  PyMyArray* self = (PyMyArray*)obj;
   view->obj = (PyObject*)self;
   view->buf = (void*)self->arr.arr;
   view->len = self->arr.length * sizeof(int);
@@ -218,7 +224,6 @@
 #ifdef __GNUC__
 extern __attribute__((visibility("default")))
 #else
-extern __declspec(dllexport)
 #endif
 
 PyMODINIT_FUNC
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -87,4 +87,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -18,7 +18,8 @@
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER)
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -386,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -814,8 +817,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
diff --git a/pypy/module/test_lib_pypy/test_gdbm_extra.py b/pypy/module/test_lib_pypy/test_gdbm_extra.py
--- a/pypy/module/test_lib_pypy/test_gdbm_extra.py
+++ b/pypy/module/test_lib_pypy/test_gdbm_extra.py
@@ -15,3 +15,7 @@
     assert len(g) == 2
     del g['abc']
     assert len(g) == 1
+
+def test_unicode():
+    path = unicode(udir.join('test_gdm_unicode'))
+    g = gdbm.open(path, 'c')  # does not crash

From pypy.commits at gmail.com  Fri Aug 26 01:20:14 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 25 Aug 2016 22:20:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge cpyext-subclass which copies
 flags when inheriting
Message-ID: <57bfd18e.d32d1c0a.3ca21.ebb3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86541:462a7ade93b6
Date: 2016-08-26 13:01 +1000
http://bitbucket.org/pypy/pypy/changeset/462a7ade93b6/

Log:	merge cpyext-subclass which copies flags when inheriting

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,7 +120,7 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 """.split()
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -87,4 +87,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -18,7 +18,8 @@
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER)
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -386,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -814,8 +817,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 

From pypy.commits at gmail.com  Fri Aug 26 01:30:07 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Thu, 25 Aug 2016 22:30:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: cleanup
Message-ID: <57bfd3df.04141c0a.e63b0.f944@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r86544:6addab7e0fb1
Date: 2016-08-25 22:28 -0700
http://bitbucket.org/pypy/pypy/changeset/6addab7e0fb1/

Log:	cleanup

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1234,10 +1234,7 @@
         if d.values:
             for i in range(len(d.values)):
                 key = d.keys[i]
-                if key is None:
-                    is_unpacking = True
-                else:
-                    is_unpacking = False
+                is_unpacking = key is None
                 if elements == 0xFFFF or (elements and is_unpacking):
                     self.emit_op_arg(ops.BUILD_MAP, elements)
                     containers += 1
@@ -1265,7 +1262,7 @@
             oparg = min(containers, 255)
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
-            is_unpacking = 0
+            is_unpacking = False
     
     def visit_Set(self, s):
         self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)

From pypy.commits at gmail.com  Fri Aug 26 05:18:42 2016
From: pypy.commits at gmail.com (vext01)
Date: Fri, 26 Aug 2016 02:18:42 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Fix type errors exposed
 by backend tests.
Message-ID: <57c00972.17a61c0a.dead8.4045@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86545:2cbba6120432
Date: 2016-08-26 10:16 +0100
http://bitbucket.org/pypy/pypy/changeset/2cbba6120432/

Log:	Fix type errors exposed by backend tests.

diff --git a/rpython/jit/backend/llsupport/gcreftracer.py b/rpython/jit/backend/llsupport/gcreftracer.py
--- a/rpython/jit/backend/llsupport/gcreftracer.py
+++ b/rpython/jit/backend/llsupport/gcreftracer.py
@@ -44,10 +44,12 @@
 def make_boehm_tracer(array_base_addr, gcrefs):
     # copy the addresses, but return 'gcrefs' as the object that must be
     # kept alive
+    assert isinstance(array_base_addr, int)
+    array_base_addr_p = rffi.cast(rffi.CCHARP, array_base_addr)
     n_gcrefs = len(gcrefs)
-    set_pages_writable(array_base_addr, n_gcrefs * WORD)
+    set_pages_writable(array_base_addr_p, n_gcrefs * WORD)
     for i in range(n_gcrefs):
         p = rffi.cast(rffi.SIGNEDP, array_base_addr + i * WORD)
         p[0] = rffi.cast(lltype.Signed, gcrefs[i])
-    set_pages_executable(array_base_addr, n_gcrefs * WORD)
+    set_pages_executable(array_base_addr_p, n_gcrefs * WORD)
     return gcrefs
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -727,14 +727,20 @@
         return c_mprotect_safe(addr, size, prot)
 
     def set_pages_executable(addr, size):
+        assert isinstance(addr, lltype._ptr)
+        assert isinstance(size, int)
         rv = mprotect(addr, size, PROT_EXEC | PROT_READ)
-        if rv < 0:
+        if int(rv) < 0:
+            from rpython.rlib import debug
             debug.fatalerror_notb("set_pages_executable failed")
 
     def set_pages_writable(addr, size):
+        assert isinstance(addr, lltype._ptr)
+        assert isinstance(size, int)
         rv = mprotect(addr, size, PROT_WRITE | PROT_READ)
-        if rv < 0:
-            debug.fatalerror_notb("set_pages_executable failed")
+        if int(rv) < 0:
+            from rpython.rlib import debug
+            debug.fatalerror_notb("set_pages_writable failed")
 
     def clear_large_memory_chunk_aligned(addr, map_size):
         addr = rffi.cast(PTR, addr)

From pypy.commits at gmail.com  Fri Aug 26 05:18:44 2016
From: pypy.commits at gmail.com (vext01)
Date: Fri, 26 Aug 2016 02:18:44 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Mark or comment tests
 which fail on OpenBSD (probably unrelated to W^X).
Message-ID: <57c00974.08d11c0a.66df4.3e71@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86546:2bcb6534dd1c
Date: 2016-08-26 10:17 +0100
http://bitbucket.org/pypy/pypy/changeset/2bcb6534dd1c/

Log:	Mark or comment tests which fail on OpenBSD (probably unrelated to
	W^X).

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -4481,6 +4481,11 @@
                 expected = heaptracker.int_signext(test_case, numbytes)
                 assert got == expected
 
+    @pytest.mark.xfail(sys.platform.startswith("openbsd"),
+                       reason="emits wrong code, or objdump reports wrong."
+                       "looks like openbsd objdump doesn't understand NOPs "
+                       "with operands. e.g. 0f1f00 is 'nop DWORD PTR [rax]', "
+                       "but we just get '(bad)'. Old binutils issue?")
     def test_compile_asmlen(self):
         if not isinstance(self.cpu, AbstractLLCPU):
             py.test.skip("pointless test on non-asm")
diff --git a/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
@@ -3,6 +3,8 @@
 from rpython.jit.backend.x86.test import test_rx86_32_auto_encoding
 
 
+# XXX OpenBSD
+# Many unrecognised asm instrs. Probably due to openbsd's old binutils
 class TestRx86_64(test_rx86_32_auto_encoding.TestRx86_32):
     WORD = 8
     TESTDIR = 'rx86_64'
diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py
--- a/rpython/jit/backend/x86/test/test_zrpy_gc.py
+++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py
@@ -1,6 +1,16 @@
 from rpython.jit.backend.llsupport.test.zrpy_gc_test import CompileFrameworkTests
+import pytest
 
 
+# XXX OpenBSD
+# Generated binary crashes:
+# mem.c: 13 mallocs left (use PYPY_ALLOC=1 to see the list)
+# RPython traceback:
+#   File "rpython_jit_backend_llsupport_test.c", line 232, in entrypoint
+#   File "rpython_jit_backend_llsupport_test.c", line 520, in allfuncs
+#   File "rpython_jit_backend_llsupport_test.c", line 599, in main_allfuncs
+#   File "rpython_rtyper_lltypesystem.c", line 3658, in ll_dict_getitem_with_hash__dicttablePtr_rpy_stri
+# Fatal RPython error: KeyError
 class TestShadowStack(CompileFrameworkTests):
     gcrootfinder = "shadowstack"
     gc = "incminimark"
diff --git a/rpython/jit/backend/zarch/test/test_auto_encoding.py b/rpython/jit/backend/zarch/test/test_auto_encoding.py
--- a/rpython/jit/backend/zarch/test/test_auto_encoding.py
+++ b/rpython/jit/backend/zarch/test/test_auto_encoding.py
@@ -285,6 +285,9 @@
         if not data.startswith('GNU assembler'):
             py.test.skip("full tests require the GNU 'as' assembler")
 
+
+    # XXX OpenBSD
+    # Many unrecognised asm instrs. Probably due to openbsd's old binutils
     @py.test.mark.parametrize("name", codebuilder.all_instructions)
     def test_all(self, name):
         self.complete_test(name)

From pypy.commits at gmail.com  Fri Aug 26 05:19:16 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 26 Aug 2016 02:19:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: first tuple indexing test
 passed
Message-ID: <57c00994.031dc20a.f5e91.10da@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86547:141288afb1ce
Date: 2016-08-26 11:18 +0200
http://bitbucket.org/pypy/pypy/changeset/141288afb1ce/

Log:	first tuple indexing test passed

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -83,16 +83,31 @@
         start = 0
 
         view = self.buf
-        length = space.len(w_tuple)
-        for i, w_obj in enumerate(w_tuple.getitems_unroll()):
-            value = w_obj.int_w(space)
-            start = self.lookup_dimension(start, dim, index)
+        length = space.len_w(w_tuple)
+        dim = view.getndim()
+        dim = 0
+        while dim < length:
+            w_obj = w_tuple.getitem(space, dim)
+            index = w_obj.int_w(space)
+            start = self.lookup_dimension(space, start, dim, index)
+            dim += 1
         return start
 
     def lookup_dimension(self, space, start, dim, index):
+        view = self.buf
+        shape = view.getshape()
+        strides = view.getstrides()
+        nitems = shape[dim]
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim+1)
+        start += strides[dim] * index
+        # TODO suboffsets?
         return start
 
-    def _getitem_tuple_indexed(self, w_index):
+    def _getitem_tuple_indexed(self, space, w_index):
         view = self.buf
 
         fmt = view.getformat() # TODO adjust format?
@@ -100,11 +115,11 @@
         length = space.len_w(w_index)
         ndim = view.getndim()
         if length < ndim:
-            raise OperationError(self.w_NotImplementedError, \
-                    self.wrap("sub-views are not implemented"))
+            raise OperationError(space.w_NotImplementedError, \
+                    space.wrap("sub-views are not implemented"))
 
         if length > ndim:
-            raise oefmt(self.w_NotImplementedError, \
+            raise oefmt(space.w_NotImplementedError, \
                     "cannot index %d-dimension view with %d-element tuple",
                     length, ndim)
 
@@ -119,7 +134,7 @@
     def descr_getitem(self, space, w_index):
         self._check_released(space)
 
-        if self.isinstance_w(w_index, self.w_tuple):
+        if space.isinstance_w(w_index, space.w_tuple):
             return self._getitem_tuple_indexed(space, w_index)
 
         start, stop, step, size = space.decode_index4_or_tuple_index(w_index, \
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -1,3 +1,9 @@
+import struct
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.typedef import TypeDef
+from rpython.rlib.buffer import Buffer
+
 class AppTestMemoryView:
     spaceconfig = dict(usemodules=['array'])
 
@@ -166,12 +172,89 @@
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
 
-from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+class MockBuffer(Buffer):
+    def __init__(self, space, w_arr, w_dim, w_fmt, \
+                 w_itemsize, w_strides, w_shape):
+        self.space = space
+        self.w_arr = w_arr
+        self.arr = []
+        self.ndim = space.int_w(w_dim)
+        self.fmt = space.str_w(w_fmt)
+        self.itemsize = space.int_w(w_itemsize)
+        self.strides = []
+        for w_i in w_strides.getitems_unroll():
+            self.strides.append(space.int_w(w_i))
+        self.shape = []
+        for w_i in w_shape.getitems_unroll():
+            self.shape.append(space.int_w(w_i))
+        self.readonly = True
+        self.shape.append(space.len_w(w_arr))
+        self.data = []
+        itemsize = 1
+        for i, w_obj in enumerate(w_arr.getitems_unroll()):
+            ints = []
+            for j, w_i in enumerate(w_obj.getitems_unroll()):
+                ints.append(space.int_w(w_i))
+                self.data.append(space.int_w(w_i))
+            self.arr.append(ints)
 
-class AppTestMemoryViewMicroNumPyPy(BaseNumpyAppTest):
-    spaceconfig = dict(usemodules=['array', 'micronumpy'])
+    def getitem(self, index):
+        return struct.pack(self.fmt, self.data[index])[0]
+
+    def getlength(self):
+        return len(self.data)
+
+    def getitemsize(self):
+        return self.itemsize
+
+    def getndim(self):
+        return self.ndim
+
+    def getstrides(self):
+        return self.strides
+
+    def getshape(self):
+        return self.shape
+
+class W_MockArray(W_Root):
+    def __init__(self, w_list, w_dim, w_fmt, w_size, w_strides, w_shape):
+        self.w_list = w_list
+        self.w_dim = w_dim
+        self.w_fmt = w_fmt
+        self.w_size = w_size
+        self.w_strides = w_strides
+        self.w_shape = w_shape
+
+    @staticmethod
+    def descr_new(space, w_type, w_list, w_dim, w_fmt, \
+                         w_size, w_strides, w_shape):
+        return W_MockArray(w_list, w_dim, w_fmt, w_size, w_strides, w_shape)
+
+    def buffer_w(self, space, flags):
+        return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
+                          self.w_size, self.w_strides, self.w_shape)
+
+W_MockArray.typedef = TypeDef("MockArray",
+    __new__ = interp2app(W_MockArray.descr_new),
+)
+
+from pypy.objspace.std.transparent import register_proxyable
+from pypy.conftest import option
+
+class AppTestMemoryViewMicroNumPyPy(object):
+    spaceconfig = dict(usemodules=[])
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("Impossible to run on appdirect")
+        cls.w_MockArray = cls.space.gettypefor(W_MockArray)
 
     def test_tuple_indexing(self):
-        from numpy import ndarray
-        content = ndarray(list(range(12))).reshape(3,4)
-        assert memoryview(content)[0,0] == 0
+        content = self.MockArray([[0,1,2,3], [4,5,6,7], [8,9,10,11]],
+                                 dim=2, fmt='B', size=1,
+                                 strides=[4,1], shape=[3,4])
+        view = memoryview(content)
+        assert view[0,0] == 0
+        assert view[2,0] == 8
+        assert view[2,3] == 11
+        assert view[-1,-1] == 11
+        assert view[-3,-4] == 0

From pypy.commits at gmail.com  Fri Aug 26 06:44:31 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 26 Aug 2016 03:44:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: add a new test for a 3 dim,
 non byte sized memory view
Message-ID: <57c01d8f.17a71c0a.5f076.6927@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86548:21fedb5e1011
Date: 2016-08-26 12:43 +0200
http://bitbucket.org/pypy/pypy/changeset/21fedb5e1011/

Log:	add a new test for a 3 dim, non byte sized memory view

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -119,13 +119,13 @@
                     space.wrap("sub-views are not implemented"))
 
         if length > ndim:
-            raise oefmt(space.w_NotImplementedError, \
+            raise oefmt(space.w_TypeError, \
                     "cannot index %d-dimension view with %d-element tuple",
                     length, ndim)
 
         start = self._start_from_tuple(space, w_index)
 
-        buf = SubBuffer(self.buf, start, self.itemsize)
+        buf = SubBuffer(self.buf, start, view.getitemsize())
         fmtiter = UnpackFormatIterator(space, buf)
         fmtiter.interpret(fmt)
         return fmtiter.result_w[0]
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -1,3 +1,4 @@
+import py
 import struct
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.gateway import interp2app
@@ -191,18 +192,34 @@
         self.shape.append(space.len_w(w_arr))
         self.data = []
         itemsize = 1
-        for i, w_obj in enumerate(w_arr.getitems_unroll()):
-            ints = []
-            for j, w_i in enumerate(w_obj.getitems_unroll()):
-                ints.append(space.int_w(w_i))
-                self.data.append(space.int_w(w_i))
-            self.arr.append(ints)
+        worklist = [(1,w_arr)]
+        while worklist:
+            dim, w_work = worklist.pop()
+            if space.isinstance_w(w_work, space.w_list):
+                for j, w_obj in enumerate(w_work.getitems_unroll()):
+                    worklist.insert(0, (dim+1, w_obj))
+                continue
+            self.data.append(space.int_w(w_work))
+
+    def getslice(self, start, stop, step, size):
+        items = []
+        bytecount = (stop - start)
+        # data is stores as list of ints, thus this gets around the
+        # issue that one cannot advance in bytes
+        count = bytecount // size
+        start = start // size
+        for i in range(start, start+count, step):
+            items.append(self.getitem(i))
+        return ''.join(items)
+
+    def getformat(self):
+        return self.fmt
 
     def getitem(self, index):
-        return struct.pack(self.fmt, self.data[index])[0]
+        return struct.pack(self.fmt, self.data[index])
 
     def getlength(self):
-        return len(self.data)
+        return len(self.data) * self.itemsize
 
     def getitemsize(self):
         return self.itemsize
@@ -258,3 +275,22 @@
         assert view[2,3] == 11
         assert view[-1,-1] == 11
         assert view[-3,-4] == 0
+
+        try:
+            view.__getitem__((2**63-1,0))
+            assert False, "must not succeed"
+        except IndexError: pass
+
+        try:
+            view.__getitem__((0, 0, 0))
+            assert False, "must not succeed"
+        except TypeError: pass
+
+    def test_tuple_indexing_int(self):
+        content = self.MockArray([ [[1],[2],[3]], [[4],[5],[6]] ],
+                                 dim=3, fmt='i', size=4,
+                                 strides=[12,4,4], shape=[2,3,1])
+        view = memoryview(content)
+        assert view[0,0,0] == 1
+        assert view[-1,2,0] == 6
+

From pypy.commits at gmail.com  Fri Aug 26 07:32:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 04:32:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: pep446: Make newly created
 file descriptors non-inheritable
Message-ID: <57c028bd.a699c20a.32011.4ca9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86549:12d2da7409ba
Date: 2016-08-26 13:30 +0200
http://bitbucket.org/pypy/pypy/changeset/12d2da7409ba/

Log:	pep446: Make newly created file descriptors non-inheritable


From pypy.commits at gmail.com  Fri Aug 26 07:32:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 04:32:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Remove file,
 probably added during a merge by mistake
Message-ID: <57c028bf.271ac20a.51b3b.4055@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86550:3b5504537968
Date: 2016-08-26 13:31 +0200
http://bitbucket.org/pypy/pypy/changeset/3b5504537968/

Log:	Remove file, probably added during a merge by mistake

diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py
deleted file mode 100644
--- a/pypy/module/_file/readinto.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import sys, errno
-from rpython.rlib import rposix
-from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._file.interp_file import is_wouldblock_error, signal_checker
-
-_WIN32 = sys.platform.startswith('win')
-UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
-
-os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read',
-                          [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
-                          rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO)
-
-
-def direct_readinto(self, w_rwbuffer):
-    rwbuffer = self.space.writebuf_w(w_rwbuffer)
-    stream = self.getstream()
-    size = rwbuffer.getlength()
-    target_address = lltype.nullptr(rffi.CCHARP.TO)
-    fd = -1
-    target_pos = 0
-
-    if size > 64:
-        try:
-            target_address = rwbuffer.get_raw_address()
-        except ValueError:
-            pass
-        else:
-            fd = stream.try_to_find_file_descriptor()
-
-    if fd < 0 or not target_address:
-        # fall-back
-        MAX_PART = 1024 * 1024    # 1 MB
-        while size > MAX_PART:
-            data = self.direct_read(MAX_PART)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-            if len(data) != MAX_PART:
-                break
-        else:
-            data = self.direct_read(size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-
-    else:
-        # optimized case: reading more than 64 bytes into a rwbuffer
-        # with a valid raw address
-        self.check_readable()
-
-        # first "read" the part that is already sitting in buffers, if any
-        initial_size = min(size, stream.count_buffered_bytes())
-        if initial_size > 0:
-            data = stream.read(initial_size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-
-        # then call os_read() to get the rest
-        if size > 0:
-            stream.flush()
-            while True:
-                got = os_read(fd, rffi.ptradd(target_address, target_pos), size)
-                got = rffi.cast(lltype.Signed, got)
-                if got > 0:
-                    target_pos += got
-                    size -= got
-                    if size <= 0:
-                        break
-                elif got == 0:
-                    break
-                else:
-                    err = rposix.get_saved_errno()
-                    if err == errno.EINTR:
-                        signal_checker(self.space)()
-                        continue
-                    if is_wouldblock_error(err) and target_pos > 0:
-                        break
-                    raise OSError(err, "read error")
-            keepalive_until_here(rwbuffer)
-
-    return self.space.wrap(target_pos)

From pypy.commits at gmail.com  Fri Aug 26 08:15:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 26 Aug 2016 05:15:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: working on memoryview cast.
 some of that logic should move the the rlib buffer
Message-ID: <57c032f5.c75dc20a.8a59a.6086@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86551:5053ecabfdc0
Date: 2016-08-26 14:15 +0200
http://bitbucket.org/pypy/pypy/changeset/5053ecabfdc0/

Log:	working on memoryview cast. some of that logic should move the the
	rlib buffer

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -309,6 +309,7 @@
         return size
 
     def _zero_in_shape(self):
+        # TODO move to buffer
         view = self.buf
         shape = view.shape
         for i in range(view.ndim):
@@ -320,23 +321,25 @@
         # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
         self._check_released(space)
 
-        if not space.isinstance_w(w_obj, space.w_unicode):
+        if not space.isinstance_w(w_format, space.w_unicode):
             raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: format argument must be a string"))
 
-        buf = self.buf
+        fmt = space.str_w(w_format)
+        view = self.buf
         ndim = 1
 
-        if not memory_view_c_contiguous(buf.flags):
+        if not memory_view_c_contiguous(space, view.flags):
             raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: casts are restricted" \
                                " to C-contiguous views"))
 
-        if (w_shape or buf.ndim != 1) and self._zero_in_shape():
+        if (w_shape or view.getndim() != 1) and self._zero_in_shape():
             raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: cannot casts view with" \
                                " zeros in shape or strides"))
 
+        itemsize = self.get_native_fmtchar(fmt)
         if w_shape:
             if not (space.is_w(w_obj, space.w_list) or space.is_w(w_obj, space.w_tuple)):
                 raise oefmt(space.w_TypeError, "expected list or tuple got %T", w_obj)
@@ -349,19 +352,13 @@
                 raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: cast must be 1D -> ND or ND -> 1D"))
 
-        mv = W_MemoryView(self.buf)
-        mv._init_shared_values(space, self)
-        mv.itemsize = self.get_native_fmtchar(fmt)
+            shape = [space.int_w(w_obj) for w_obj in w_shape.fixedview_unroll()]
+            return W_MemoryView(Buffer.cast_to(buf, itemsize, shape))
 
-        if not mv._cast_to_1D(space, fmt):
-            return space.w_None
-        if w_shape is not space.w_None:
-            shape = space.fixedview(w_shape)
-            if not mv._cast_to_ND(space, shape, ndim):
-                return space.w_None
-        return mv
+        return W_MemoryView(Buffer.cast_to(buf, itemsize, None))
 
     def _init_flags(self):
+        # TODO move to buffer.py
         view = self.buf
         ndim = view.ndim
         flags = 0
@@ -381,30 +378,31 @@
 
     def _cast_to_1D(self, space, fmt):
         itemsize = self.get_native_fmtchar(fmt)
-        view = self.buf
+        buf = self.buf
         if itemsize < 0:
             raise OperationError(space.w_ValueError, "memoryview: destination" \
                     " format must be a native single character format prefixed" \
                     " with an optional '@'")
 
-        if self.get_native_fmtchar(view.format) < 0 or \
-           (not is_byte_format(fmt) and not is_byte_format(view.format)):
+        buffmt = buf.getformat()
+        if self.get_native_fmtchar(buffmt) < 0 or \
+           (not is_byte_format(fmt) and not is_byte_format(buffmt)):
             raise OperationError(space.w_TypeError,
                     "memoryview: cannot cast between" \
                     " two non-byte formats")
 
-        if view.length % itemsize != 0:
+        if buf.getlength() % itemsize != 0:
             raise OperationError(space.w_TypeError,
                     "memoryview: length is not a multiple of itemsize")
 
-        view.format = get_native_fmtstr(fmt)
-        if not view.format:
+        buf.format = get_native_fmtstr(fmt)
+        if not buffmt:
             raise OperationError(space.w_RuntimeError,
                     "memoryview: internal error")
-        view.itemsize = itemsize
-        view.ndim = 1
-        view.shape[0] = view.length / view.itemsize
-        view.srides[0] = view.itemsize
+        buf.itemsize = itemsize
+        buf.ndim = 1
+        buf.shape[0] = buf.length / buf.itemsize
+        buf.srides[0] = buf.itemsize
         # XX suboffsets
 
         mv._init_flags()
@@ -412,13 +410,6 @@
     def _cast_to_ND(self, space, shape, ndim):
         pass
 
-    def _init_shared_values(self, space, of):
-        mv.buf = buf # XXX not quite right
-        mv.format = of.format
-        mv.readonly = of.readonly
-        mv.itemsize = of.itemsize
-        return mv
-
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         self._check_released(space)
@@ -427,8 +418,8 @@
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'
 
-def memory_view_c_contiguous(flags):
-    return flags & (space.BUF_CONTIG_RO|space.BUF_C) != 0
+def memory_view_c_contiguous(space, flags):
+    return flags & (space.BUF_CONTIG_RO|space.MEMORYVIEW_C) != 0
 
 W_MemoryView.typedef = TypeDef(
     "memoryview",
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -177,6 +177,7 @@
     def __init__(self, space, w_arr, w_dim, w_fmt, \
                  w_itemsize, w_strides, w_shape):
         self.space = space
+        self.flags = space.MEMORYVIEW_C
         self.w_arr = w_arr
         self.arr = []
         self.ndim = space.int_w(w_dim)
@@ -294,3 +295,9 @@
         assert view[0,0,0] == 1
         assert view[-1,2,0] == 6
 
+    def test_cast_empty(self):
+        empty = self.MockArray([], dim=1, fmt='i', size=4, strides=[1], shape=[1])
+        view = memoryview(empty)
+        cview = view.cast('i')
+        assert cview.tobytes() == b''
+

From pypy.commits at gmail.com  Fri Aug 26 08:16:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 05:16:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: Implement the basics
Message-ID: <57c0331e.8aacc20a.f3495.55db@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86552:6ba4a5df9526
Date: 2016-08-26 14:15 +0200
http://bitbucket.org/pypy/pypy/changeset/6ba4a5df9526/

Log:	Implement the basics

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -2080,13 +2080,45 @@
 
 eci_inheritable = eci.merge(ExternalCompilationInfo(
     separate_module_sources=["""
+#include <errno.h>
+
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)
 {
-    /* XXX minimal impl. XXX */
-    int request = inheritable ? FIONCLEX : FIOCLEX;
-    return ioctl(fd, request, NULL);
+    static int ioctl_works = -1;
+    int flags;
+
+    if (ioctl_works != 0) {
+        int request = inheritable ? FIONCLEX : FIOCLEX;
+        int err = ioctl(fd, request, NULL);
+        if (!err) {
+            ioctl_works = 1;
+            return 0;
+        }
+
+        if (errno != ENOTTY && errno != EACCES) {
+            return -1;
+        }
+        else {
+            /* ENOTTY: The ioctl is declared but not supported by the
+               kernel.  EACCES: SELinux policy, this can be the case on
+               Android. */
+            ioctl_works = 0;
+        }
+        /* fallback to fcntl() if ioctl() does not work */
+    }
+
+    flags = fcntl(fd, F_GETFD);
+    if (flags < 0)
+        return -1;
+
+    if (inheritable)
+        flags &= ~FD_CLOEXEC;
+    else
+        flags |= FD_CLOEXEC;
+    return fcntl(fd, F_SETFD, flags);
 }
+
 RPY_EXTERN
 int rpy_get_inheritable(int fd)
 {
@@ -2106,10 +2138,25 @@
                              compilation_info=eci_inheritable)
 
 def set_inheritable(fd, inheritable):
-    error = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', error)
+    result = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', result)
 
 def get_inheritable(fd):
     res = c_get_inheritable(fd)
     res = handle_posix_error('get_inheritable', res)
     return res != 0
+
+class SetNonInheritableCache(object):
+    """Make one prebuilt instance of this for each path that creates
+    file descriptors, where you don't necessarily know if that function
+    returns inheritable or non-inheritable file descriptors.
+    """
+    _immutable_fields_ = ['cached_inheritable?']
+    cached_inheritable = -1    # -1 = don't know yet; 0 = off; 1 = on
+
+    def set_non_inheritable(self, fd):
+        if self.cached_inheritable == -1:
+            self.cached_inheritable = get_inheritable(fd)
+        if self.cached_inheritable == 1:
+            # 'fd' is inheritable; we must manually turn it off
+            set_inheritable(fd, False)
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -589,3 +589,18 @@
     assert rposix.get_inheritable(fd1) == False
     os.close(fd1)
     os.close(fd2)
+
+def test_SetNonInheritableCache():
+    cache = rposix.SetNonInheritableCache()
+    fd1, fd2 = os.pipe()
+    assert rposix.get_inheritable(fd1) == True
+    assert rposix.get_inheritable(fd1) == True
+    assert cache.cached_inheritable == -1
+    cache.set_non_inheritable(fd1)
+    assert cache.cached_inheritable == 1
+    cache.set_non_inheritable(fd2)
+    assert cache.cached_inheritable == 1
+    assert rposix.get_inheritable(fd1) == False
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)

From pypy.commits at gmail.com  Fri Aug 26 08:47:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 05:47:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: Make pipe() return
 non-inheritable fds using the now-official way. Add
Message-ID: <57c03a68.04141c0a.e63b0.99f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86553:bbc604db574e
Date: 2016-08-26 14:46 +0200
http://bitbucket.org/pypy/pypy/changeset/bbc604db574e/

Log:	Make pipe() return non-inheritable fds using the now-official way.
	Add pipe2().

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -76,6 +76,8 @@
         'device_encoding': 'interp_posix.device_encoding',
 
         'scandir': 'interp_scandir.scandir',
+        'get_inheritable': 'interp_posix.get_inheritable',
+        'set_inheritable': 'interp_posix.set_inheritable',
     }
 
     if hasattr(os, 'chown'):
@@ -193,6 +195,9 @@
     interpleveldefs['_have_functions'] = (
         'space.newlist([space.wrap(x) for x in interp_posix.have_functions])')
 
+    if rposix.HAVE_PIPE2:
+        interpleveldefs['pipe2'] = 'interp_posix.pipe2'
+
     def startup(self, space):
         from pypy.module.posix import interp_posix
         from pypy.module.imp import importing
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -889,15 +889,32 @@
             result_w[i] = space.fsdecode(w_bytes)
     return space.newlist(result_w)
 
+ at unwrap_spec(fd=c_int)
+def get_inheritable(space, fd):
+    return space.wrap(rposix.get_inheritable(fd))
+
+ at unwrap_spec(fd=c_int, inheritable=int)
+def set_inheritable(space, fd, inheritable):
+    rposix.set_inheritable(fd, inheritable)
+
+_pipe_inhcache = rposix.SetNonInheritableCache()
+
 def pipe(space):
     "Create a pipe.  Returns (read_end, write_end)."
     try:
-        fd1, fd2 = os.pipe()
+        fd1, fd2 = rposix.pipe(rposix.O_CLOEXEC or 0)
+        _pipe_inhcache.set_non_inheritable(fd1)
+        _pipe_inhcache.set_non_inheritable(fd2)
     except OSError as e:
         raise wrap_oserror(space, e)
-    # XXX later, use rposix.pipe2() if available!
-    rposix.set_inheritable(fd1, False)
-    rposix.set_inheritable(fd2, False)
+    return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
+
+ at unwrap_spec(flags=c_int)
+def pipe2(space, flags):
+    try:
+        fd1, fd2 = rposix.pipe2(flags)
+    except OSError as e:
+        raise wrap_oserror(space, e)
     return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1077,6 +1077,33 @@
         x = f.read(1)
         assert x == 'e'
 
+    def test_pipe_inheritable(self):
+        fd1, fd2 = self.posix.pipe()
+        assert self.posix.get_inheritable(fd1) == False
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_pipe2(self):
+        if not hasattr(self.posix, 'pipe2'):
+            skip("no pipe2")
+        fd1, fd2 = self.posix.pipe2(0)
+        assert self.posix.get_inheritable(fd1) == True
+        assert self.posix.get_inheritable(fd2) == True
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_O_CLOEXEC(self):
+        if not hasattr(self.posix, 'pipe2'):
+            skip("no pipe2")
+        if not hasattr(self.posix, 'O_CLOEXEC'):
+            skip("no O_CLOEXEC")
+        fd1, fd2 = self.posix.pipe2(self.posix.O_CLOEXEC)
+        assert self.posix.get_inheritable(fd1) == False
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
     def test_urandom(self):
         os = self.posix
         s = os.urandom(5)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1113,37 +1113,74 @@
     c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
                                                     rffi.INT],
                                 rffi.INT)
+    HAVE_PIPE2 = False
+    O_CLOEXEC = None
 else:
     INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
     c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT,
                       save_err=rffi.RFFI_SAVE_ERRNO)
+    class CConfig:
+        _compilation_info_ = eci
+        HAVE_PIPE2 = rffi_platform.Has('pipe2')
+        O_CLOEXEC = rffi_platform.DefinedConstantInteger('O_CLOEXEC')
+    config = rffi_platform.configure(CConfig)
+    HAVE_PIPE2 = config['HAVE_PIPE2']
+    O_CLOEXEC = config['O_CLOEXEC']
+    if HAVE_PIPE2:
+        c_pipe2 = external('pipe2', [INT_ARRAY_P, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('pipe')
-def pipe():
+def pipe(flags=0):
+    # 'flags' might be ignored.  Check the result.
     if _WIN32:
+        # 'flags' ignored
         pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         try:
-            if not CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                raise WindowsError(rwin32.GetLastError_saved(),
-                                   "CreatePipe failed")
+            ok = CreatePipe(
+                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0)
             hread = rffi.cast(rffi.INTPTR_T, pread[0])
             hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
         finally:
             lltype.free(pwrite, flavor='raw')
             lltype.free(pread, flavor='raw')
-        fdread = c_open_osfhandle(hread, 0)
-        fdwrite = c_open_osfhandle(hwrite, 1)
+        if ok:
+            fdread = c_open_osfhandle(hread, 0)
+            fdwrite = c_open_osfhandle(hwrite, 1)
+            if fdread == -1 or fdwrite == -1:
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
+                ok = 0
+        if not ok:
+            raise WindowsError(rwin32.GetLastError_saved(),
+                               "CreatePipe failed")
         return (fdread, fdwrite)
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
-            handle_posix_error('pipe', c_pipe(filedes))
+            if HAVE_PIPE2:
+                res = c_pipe2(filedes, flags)
+                if widen(res) != 0 and get_saved_errno() == errno.ENOSYS:
+                    res = c_pipe(filedes)
+            else:
+                res = c_pipe(filedes)      # 'flags' ignored
+            handle_posix_error('pipe', res)
             return (widen(filedes[0]), widen(filedes[1]))
         finally:
             lltype.free(filedes, flavor='raw')
 
+def pipe2(flags):
+    # Only available if there is really a c_pipe2 function.
+    # No fallback to pipe() if we get ENOSYS.
+    filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
+    try:
+        res = c_pipe2(filedes, flags)
+        handle_posix_error('pipe2', res)
+        return (widen(filedes[0]), widen(filedes[1]))
+    finally:
+        lltype.free(filedes, flavor='raw')
+
 c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
                   save_err=rffi.RFFI_SAVE_ERRNO,)
 c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT,

From pypy.commits at gmail.com  Fri Aug 26 09:23:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 06:23:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: os.dup(), os.dup2()
Message-ID: <57c042ed.8a13c20a.23ed6.70b0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86554:1b8d1ba5dcb8
Date: 2016-08-26 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/1b8d1ba5dcb8/

Log:	os.dup(), os.dup2()

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -536,17 +536,17 @@
     """Create a copy of the file descriptor.  Return the new file
 descriptor."""
     try:
-        newfd = os.dup(fd)
+        newfd = rposix.dup(fd, inheritable=False)
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
         return space.wrap(newfd)
 
- at unwrap_spec(old_fd=c_int, new_fd=c_int)
-def dup2(space, old_fd, new_fd):
+ at unwrap_spec(old_fd=c_int, new_fd=c_int, inheritable=int)
+def dup2(space, old_fd, new_fd, inheritable=1):
     """Duplicate a file descriptor."""
     try:
-        os.dup2(old_fd, new_fd)
+        rposix.dup2(old_fd, new_fd, inheritable)
     except OSError as e:
         raise wrap_oserror(space, e)
 
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -106,6 +106,7 @@
         posix = self.posix
         fd = posix.open(path, posix.O_RDONLY, 0o777)
         fd2 = posix.dup(fd)
+        assert posix.get_inheritable(fd2) == False
         assert not posix.isatty(fd2)
         s = posix.read(fd, 1)
         assert s == b't'
@@ -1104,6 +1105,18 @@
         self.posix.close(fd1)
         self.posix.close(fd2)
 
+    def test_dup2_inheritable(self):
+        fd1, fd2 = self.posix.pipe()
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.dup2(fd1, fd2)
+        assert self.posix.get_inheritable(fd2) == True
+        self.posix.dup2(fd1, fd2, False)
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.dup2(fd1, fd2, True)
+        assert self.posix.get_inheritable(fd2) == True
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
     def test_urandom(self):
         os = self.posix
         s = os.urandom(5)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -366,14 +366,22 @@
     return result
 
 @replace_os_function('dup')
-def dup(fd):
+def dup(fd, inheritable=True):
     validate_fd(fd)
-    return handle_posix_error('dup', c_dup(fd))
+    if inheritable:
+        res = c_dup(fd)
+    else:
+        res = c_dup_noninheritable(fd)
+    return handle_posix_error('dup', res)
 
 @replace_os_function('dup2')
-def dup2(fd, newfd):
+def dup2(fd, newfd, inheritable=True):
     validate_fd(fd)
-    handle_posix_error('dup2', c_dup2(fd, newfd))
+    if inheritable:
+        res = c_dup2(fd, newfd)
+    else:
+        res = c_dup2_noninheritable(fd, newfd)
+    handle_posix_error('dup2', res)
 
 #___________________________________________________________________
 
@@ -1114,6 +1122,7 @@
                                                     rffi.INT],
                                 rffi.INT)
     HAVE_PIPE2 = False
+    HAVE_DUP3 = False
     O_CLOEXEC = None
 else:
     INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
@@ -1122,9 +1131,11 @@
     class CConfig:
         _compilation_info_ = eci
         HAVE_PIPE2 = rffi_platform.Has('pipe2')
+        HAVE_DUP3 = rffi_platform.Has('dup3')
         O_CLOEXEC = rffi_platform.DefinedConstantInteger('O_CLOEXEC')
     config = rffi_platform.configure(CConfig)
     HAVE_PIPE2 = config['HAVE_PIPE2']
+    HAVE_DUP3 = config['HAVE_DUP3']
     O_CLOEXEC = config['O_CLOEXEC']
     if HAVE_PIPE2:
         c_pipe2 = external('pipe2', [INT_ARRAY_P, rffi.INT], rffi.INT,
@@ -2116,7 +2127,7 @@
 
 
 eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=["""
+    separate_module_sources=[r"""
 #include <errno.h>
 
 RPY_EXTERN
@@ -2164,7 +2175,60 @@
         return -1;
     return !(flags & FD_CLOEXEC);
 }
-    """],
+
+RPY_EXTERN
+int rpy_dup_noninheritable(int fd)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUPFD_CLOEXEC
+    return fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+    fd = dup(fd);
+    if (fd >= 0) {
+        if (rpy_set_inheritable(fd, 0) != 0) {
+            close(fd);
+            return -1;
+        }
+    }
+    return fd;
+#endif
+}
+
+RPY_EXTERN
+int rpy_dup2_noninheritable(int fd, int fd2)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUP2FD_CLOEXEC
+    return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
+
+#else
+# if %(HAVE_DUP3)d   /* HAVE_DUP3 */
+    static int dup3_works = -1;
+    if (dup3_works != 0) {
+        if (dup3(fd, fd2, O_CLOEXEC) >= 0)
+            return 0;
+        if (dup3_works == -1)
+            dup3_works = (errno != ENOSYS);
+        if (dup3_works)
+            return -1;
+    }
+# endif
+    if (dup2(fd, fd2) < 0)
+        return -1;
+    if (rpy_set_inheritable(fd2, 0) != 0) {
+        close(fd2);
+        return -1;
+    }
+    return 0;
+#endif
+}
+    """ % {'HAVE_DUP3': HAVE_DUP3}],
     post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
 
 c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
@@ -2173,6 +2237,12 @@
 c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
                              compilation_info=eci_inheritable)
+c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                compilation_info=eci_inheritable)
+c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                 compilation_info=eci_inheritable)
 
 def set_inheritable(fd, inheritable):
     result = c_set_inheritable(fd, inheritable)

From pypy.commits at gmail.com  Fri Aug 26 09:27:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 06:27:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fail if we call an llexternal()
 function with more arguments than declared
Message-ID: <57c043aa.0117c20a.c7a53.7cf7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86555:fda5486b3186
Date: 2016-08-26 15:26 +0200
http://bitbucket.org/pypy/pypy/changeset/fda5486b3186/

Log:	Fail if we call an llexternal() function with more arguments than
	declared

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -237,8 +237,10 @@
                              " directly to a VOIDP argument")
     _oops._annspecialcase_ = 'specialize:memo'
 
+    nb_args = len(args)
     unrolling_arg_tps = unrolling_iterable(enumerate(args))
     def wrapper(*args):
+        assert len(args) == nb_args
         real_args = ()
         # XXX 'to_free' leaks if an allocation fails with MemoryError
         # and was not the first in this function
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -49,6 +49,7 @@
         eci = ExternalCompilationInfo(includes=['stuff.h'],
                                       include_dirs=[udir])
         z = llexternal('X', [Signed], Signed, compilation_info=eci)
+        py.test.raises(AssertionError, z, 8, 9)
 
         def f():
             return z(8)

From pypy.commits at gmail.com  Fri Aug 26 09:27:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 06:27:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: Fail if we call an
 llexternal() function with more arguments than declared
Message-ID: <57c043ab.02c41c0a.78b42.e705@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86556:f1be342a8a2f
Date: 2016-08-26 15:26 +0200
http://bitbucket.org/pypy/pypy/changeset/f1be342a8a2f/

Log:	Fail if we call an llexternal() function with more arguments than
	declared

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -237,8 +237,10 @@
                              " directly to a VOIDP argument")
     _oops._annspecialcase_ = 'specialize:memo'
 
+    nb_args = len(args)
     unrolling_arg_tps = unrolling_iterable(enumerate(args))
     def wrapper(*args):
+        assert len(args) == nb_args
         real_args = ()
         # XXX 'to_free' leaks if an allocation fails with MemoryError
         # and was not the first in this function
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -49,6 +49,7 @@
         eci = ExternalCompilationInfo(includes=['stuff.h'],
                                       include_dirs=[udir])
         z = llexternal('X', [Signed], Signed, compilation_info=eci)
+        py.test.raises(AssertionError, z, 8, 9)
 
         def f():
             return z(8)

From pypy.commits at gmail.com  Fri Aug 26 09:41:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 06:41:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix: the cache didn't work on py3.5,
 because we get two different calls
Message-ID: <57c046f3.c997c20a.2d8c4.7612@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86557:37eb078dcde5
Date: 2016-08-26 15:40 +0200
http://bitbucket.org/pypy/pypy/changeset/37eb078dcde5/

Log:	Fix: the cache didn't work on py3.5, because we get two different
	calls

diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -19,7 +19,8 @@
         with open(os.path.join(lib_python, 'importlib', name + '.py')) as fp:
             source = fp.read()
         pathname = "<frozen importlib.%s>" % name
-        code_w = Module._cached_compile(space, source, pathname, 'exec', 0)
+        code_w = Module._cached_compile(space, name, source,
+                                        pathname, 'exec', 0)
         space.setitem(w_dict, space.wrap('__name__'), w_name)
         space.setitem(w_dict, space.wrap('__builtins__'),
                       space.wrap(space.builtin))
@@ -43,11 +44,11 @@
         self.w_import = space.wrap(interp_import.import_with_frames_removed)
 
     @staticmethod
-    def _cached_compile(space, source, *args):
+    def _cached_compile(space, name, source, *args):
         from rpython.config.translationoption import CACHE_DIR
         from pypy.module.marshal import interp_marshal
 
-        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_bootstrap')
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_%s' % (name,))
         try:
             if space.config.translating:
                 raise IOError("don't use the cache when translating pypy")

From pypy.commits at gmail.com  Fri Aug 26 09:56:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 06:56:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: hg merge py3.5
Message-ID: <57c04a87.43681c0a.9fdef.b3ff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86558:e1900adeea06
Date: 2016-08-26 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/e1900adeea06/

Log:	hg merge py3.5

diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -19,7 +19,8 @@
         with open(os.path.join(lib_python, 'importlib', name + '.py')) as fp:
             source = fp.read()
         pathname = "<frozen importlib.%s>" % name
-        code_w = Module._cached_compile(space, source, pathname, 'exec', 0)
+        code_w = Module._cached_compile(space, name, source,
+                                        pathname, 'exec', 0)
         space.setitem(w_dict, space.wrap('__name__'), w_name)
         space.setitem(w_dict, space.wrap('__builtins__'),
                       space.wrap(space.builtin))
@@ -43,11 +44,11 @@
         self.w_import = space.wrap(interp_import.import_with_frames_removed)
 
     @staticmethod
-    def _cached_compile(space, source, *args):
+    def _cached_compile(space, name, source, *args):
         from rpython.config.translationoption import CACHE_DIR
         from pypy.module.marshal import interp_marshal
 
-        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_bootstrap')
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_%s' % (name,))
         try:
             if space.config.translating:
                 raise IOError("don't use the cache when translating pypy")

From pypy.commits at gmail.com  Fri Aug 26 09:56:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 06:56:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: FileIO.__init__() needs to
 close the previous fd, if any
Message-ID: <57c04a89.a6a5c20a.b957f.79b5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86559:3383e4d5f747
Date: 2016-08-26 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/3383e4d5f747/

Log:	FileIO.__init__() needs to close the previous fd, if any

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -139,6 +139,7 @@
 
     @unwrap_spec(mode=str, closefd=int)
     def descr_init(self, space, w_name, mode='r', closefd=True, w_opener=None):
+        self._close(space)
         if space.isinstance_w(w_name, space.w_float):
             raise oefmt(space.w_TypeError,
                         "integer argument expected, got float")
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -246,6 +246,15 @@
             assert f.mode == 'xb'
         raises(FileExistsError, _io.FileIO, filename, 'x')
 
+    def test_close_upon_reinit(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        fd1 = f.fileno()
+        f.__init__(self.tmpfile, 'w')
+        fd2 = f.fileno()
+        if fd1 != fd2:
+            raises(OSError, posix.close, fd1)
+
 
 def test_flush_at_exit():
     from pypy import conftest

From pypy.commits at gmail.com  Fri Aug 26 09:56:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 06:56:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: io.open()
Message-ID: <57c04a8b.8f8e1c0a.2f009.bdeb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86560:2b675b70ddf7
Date: 2016-08-26 15:55 +0200
http://bitbucket.org/pypy/pypy/changeset/2b675b70ddf7/

Log:	io.open()

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -4,6 +4,7 @@
     OperationError, oefmt, wrap_oserror, wrap_oserror2)
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import rposix
 from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC, O_EXCL
 import sys, os, stat, errno
 from pypy.module._io.interp_iobase import W_RawIOBase, convert_size
@@ -29,6 +30,7 @@
 
 O_BINARY = getattr(os, "O_BINARY", 0)
 O_APPEND = getattr(os, "O_APPEND", 0)
+_open_inhcache = rposix.SetNonInheritableCache()
 
 def _bad_mode(space):
     raise oefmt(space.w_ValueError,
@@ -154,6 +156,8 @@
                 raise oefmt(space.w_ValueError, "negative file descriptor")
 
         self.readable, self.writable, self.created, self.appending, flags = decode_mode(space, mode)
+        if rposix.O_CLOEXEC is not None:
+            flags |= rposix.O_CLOEXEC
 
         fd_is_own = False
         try:
@@ -172,8 +176,7 @@
                     raise oefmt(space.w_ValueError,
                                 "Cannot use closefd=False with file name")
 
-                from pypy.module.posix.interp_posix import (
-                    dispatch_filename, rposix)
+                from pypy.module.posix.interp_posix import dispatch_filename
                 try:
                     self.fd = dispatch_filename(rposix.open)(
                         space, w_name, flags, 0666)
@@ -182,6 +185,8 @@
                                         exception_name='w_IOError')
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    _open_inhcache.set_non_inheritable(self.fd)
             else:
                 w_fd = space.call_function(w_opener, w_name, space.wrap(flags))
                 try:
@@ -193,6 +198,8 @@
                                 "expected integer from opener")
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    rposix.set_inheritable(self.fd, False)
 
             self._dircheck(space, w_name)
             space.setattr(self, space.wrap("name"), w_name)
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -246,6 +246,12 @@
             assert f.mode == 'xb'
         raises(FileExistsError, _io.FileIO, filename, 'x')
 
+    def test_non_inheritable(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        assert posix.get_inheritable(f.fileno()) == False
+        f.close()
+
     def test_close_upon_reinit(self):
         import _io, posix
         f = _io.FileIO(self.tmpfile, 'r')

From pypy.commits at gmail.com  Fri Aug 26 10:05:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:05:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: re-raise unlikely exception at
 app-level
Message-ID: <57c04cc3.c41f1c0a.12d23.bff4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86561:6d742644fb2b
Date: 2016-08-26 16:04 +0200
http://bitbucket.org/pypy/pypy/changeset/6d742644fb2b/

Log:	re-raise unlikely exception at app-level

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -186,7 +186,10 @@
                 finally:
                     fd_is_own = True
                 if not rposix._WIN32:
-                    _open_inhcache.set_non_inheritable(self.fd)
+                    try:
+                        _open_inhcache.set_non_inheritable(self.fd)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
             else:
                 w_fd = space.call_function(w_opener, w_name, space.wrap(flags))
                 try:
@@ -199,7 +202,10 @@
                 finally:
                     fd_is_own = True
                 if not rposix._WIN32:
-                    rposix.set_inheritable(self.fd, False)
+                    try:
+                        rposix.set_inheritable(self.fd, False)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
 
             self._dircheck(space, w_name)
             space.setattr(self, space.wrap("name"), w_name)

From pypy.commits at gmail.com  Fri Aug 26 10:05:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:05:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: os.open()
Message-ID: <57c04cc5.0cce1c0a.dc8d9.a4bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86562:3bab49408735
Date: 2016-08-26 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/3bab49408735/

Log:	os.open()

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -209,6 +209,8 @@
             space.w_NotImplementedError,
             "%s: %s unavailable on this platform", funcname, arg)
 
+_open_inhcache = rposix.SetNonInheritableCache()
+
 @unwrap_spec(flags=c_int, mode=c_int, dir_fd=DirFD(rposix.HAVE_OPENAT))
 def open(space, w_path, flags, mode=0777,
          __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
@@ -220,12 +222,15 @@
   and path should be relative; path will then be relative to that directory.
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
+    if rposix.O_CLOEXEC is not None:
+        flags |= rposix.O_CLOEXEC
     try:
         if rposix.HAVE_OPENAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             fd = rposix.openat(path, flags, mode, dir_fd)
         else:
             fd = dispatch_filename(rposix.open)(space, w_path, flags, mode)
+        _open_inhcache.set_non_inheritable(fd)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     return space.wrap(fd)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1117,6 +1117,13 @@
         self.posix.close(fd1)
         self.posix.close(fd2)
 
+    def test_open_inheritable(self):
+        os = self.posix
+        fd = os.open(self.path2 + 'test_open_inheritable',
+                     os.O_RDWR | os.O_CREAT, 0o666)
+        assert os.get_inheritable(fd) == False
+        os.close(fd)
+
     def test_urandom(self):
         os = self.posix
         s = os.urandom(5)

From pypy.commits at gmail.com  Fri Aug 26 10:11:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:11:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: os.fdopen()
Message-ID: <57c04e0d.eeb8c20a.90862.8086@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86563:611f7fa27fb9
Date: 2016-08-26 16:10 +0200
http://bitbucket.org/pypy/pypy/changeset/611f7fa27fb9/

Log:	os.fdopen()

diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -252,6 +252,17 @@
         assert posix.get_inheritable(f.fileno()) == False
         f.close()
 
+    def test_FileIO_fd_does_change_inheritable(self):
+        import _io, posix
+        fd1, fd2 = posix.pipe()
+        posix.set_inheritable(fd1, True)
+        f1 = _io.FileIO(fd1, 'r')
+        f2 = _io.FileIO(fd2, 'w')
+        assert posix.get_inheritable(fd1) == False
+        assert posix.get_inheritable(fd2) == True
+        f1.close()
+        f2.close()
+
     def test_close_upon_reinit(self):
         import _io, posix
         f = _io.FileIO(self.tmpfile, 'r')

From pypy.commits at gmail.com  Fri Aug 26 10:15:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:15:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: os.openpty()
Message-ID: <57c04f02.4152c20a.199e7.86a3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86564:83ec87415043
Date: 2016-08-26 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/83ec87415043/

Log:	os.openpty()

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1258,6 +1258,8 @@
     "Open a pseudo-terminal, returning open fd's for both master and slave end."
     try:
         master_fd, slave_fd = os.openpty()
+        rposix.set_inheritable(master_fd, False)
+        rposix.set_inheritable(slave_fd, False)
     except OSError as e:
         raise wrap_oserror(space, e)
     return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)])
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -399,6 +399,16 @@
             os.write(slave_fd, b'x\n')
             data = os.read(master_fd, 100)
             assert data.startswith(b'x')
+            os.close(master_fd)
+            os.close(slave_fd)
+
+        def test_openpty_non_inheritable(self):
+            os = self.posix
+            master_fd, slave_fd = os.openpty()
+            assert os.get_inheritable(master_fd) == False
+            assert os.get_inheritable(slave_fd) == False
+            os.close(master_fd)
+            os.close(slave_fd)
 
     if hasattr(__import__(os.name), "forkpty"):
         def test_forkpty(self):

From pypy.commits at gmail.com  Fri Aug 26 10:15:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:15:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Close these two file descriptors
Message-ID: <57c04f04.c2a5c20a.46c00.8723@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86565:aae784293a68
Date: 2016-08-26 16:14 +0200
http://bitbucket.org/pypy/pypy/changeset/aae784293a68/

Log:	Close these two file descriptors

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -411,6 +411,8 @@
             os.write(slave_fd, 'x\n')
             data = os.read(master_fd, 100)
             assert data.startswith('x')
+            os.close(master_fd)
+            os.close(slave_fd)
 
     if hasattr(__import__(os.name), "forkpty"):
         def test_forkpty(self):

From pypy.commits at gmail.com  Fri Aug 26 10:25:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:25:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: select.epoll()
Message-ID: <57c05145.cb7f1c0a.49c70.cbcf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86566:9ad3beb3f9d4
Date: 2016-08-26 16:24 +0200
http://bitbucket.org/pypy/pypy/changeset/9ad3beb3f9d4/

Log:	select.epoll()

diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py
--- a/pypy/module/select/interp_epoll.py
+++ b/pypy/module/select/interp_epoll.py
@@ -39,7 +39,8 @@
 for symbol in public_symbols:
     setattr(CConfig, symbol, rffi_platform.DefinedConstantInteger(symbol))
 
-for symbol in ["EPOLL_CTL_ADD", "EPOLL_CTL_MOD", "EPOLL_CTL_DEL"]:
+for symbol in ["EPOLL_CTL_ADD", "EPOLL_CTL_MOD", "EPOLL_CTL_DEL",
+               "EPOLL_CLOEXEC"]:
     setattr(CConfig, symbol, rffi_platform.ConstantInteger(symbol))
 
 cconfig = rffi_platform.configure(CConfig)
@@ -52,13 +53,14 @@
 EPOLL_CTL_ADD = cconfig["EPOLL_CTL_ADD"]
 EPOLL_CTL_MOD = cconfig["EPOLL_CTL_MOD"]
 EPOLL_CTL_DEL = cconfig["EPOLL_CTL_DEL"]
+EPOLL_CLOEXEC = cconfig["EPOLL_CLOEXEC"]
 
 DEF_REGISTER_EVENTMASK = (public_symbols["EPOLLIN"] |
                           public_symbols["EPOLLOUT"] |
                           public_symbols["EPOLLPRI"])
 
-epoll_create = rffi.llexternal(
-    "epoll_create", [rffi.INT], rffi.INT, compilation_info=eci,
+epoll_create1 = rffi.llexternal(
+    "epoll_create1", [rffi.INT], rffi.INT, compilation_info=eci,
     save_err=rffi.RFFI_SAVE_ERRNO
 )
 epoll_ctl = rffi.llexternal(
@@ -82,14 +84,12 @@
         self.epfd = epfd
         self.register_finalizer(space)
 
-    @unwrap_spec(sizehint=int)
-    def descr__new__(space, w_subtype, sizehint=-1):
-        if sizehint == -1:
-            sizehint = FD_SETSIZE - 1
-        elif sizehint < 0:
+    @unwrap_spec(sizehint=int, flags=int)
+    def descr__new__(space, w_subtype, sizehint=0, flags=0):
+        if sizehint < 0:     # 'sizehint' is otherwise ignored
             raise oefmt(space.w_ValueError,
                         "sizehint must be greater than zero, got %d", sizehint)
-        epfd = epoll_create(sizehint)
+        epfd = epoll_create1(flags | EPOLL_CLOEXEC)
         if epfd < 0:
             raise exception_from_saved_errno(space, space.w_IOError)
 
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -209,3 +209,10 @@
         ep = select.epoll()
         ep.close()
         ep.close()
+
+    def test_non_inheritable(self):
+        import select, posix
+
+        ep = select.epoll()
+        assert posix.get_inheritable(ep.fileno()) == False
+        ep.close()

From pypy.commits at gmail.com  Fri Aug 26 10:32:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:32:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: select.kqueue(),
 can't test but simple enough
Message-ID: <57c052e8.262ec20a.3aef8.9097@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86567:462212502307
Date: 2016-08-26 16:29 +0200
http://bitbucket.org/pypy/pypy/changeset/462212502307/

Log:	select.kqueue(), can't test but simple enough

diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -1,10 +1,11 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.error import exception_from_saved_errno
+from pypy.interpreter.error import exception_from_saved_errno, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.typedef import TypeDef, generic_new_descr, GetSetProperty
 from rpython.rlib._rsocket_rffi import socketclose_no_errno
 from rpython.rlib.rarithmetic import r_uint
+from rpython.rlib import rposix
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -115,6 +116,10 @@
         kqfd = syscall_kqueue()
         if kqfd < 0:
             raise exception_from_saved_errno(space, space.w_IOError)
+        try:
+            rposix.set_inheritable(kqfd, False)
+        except OSError as e:
+            raise wrap_oserror(space, e)
         return space.wrap(W_Kqueue(space, kqfd))
 
     @unwrap_spec(fd=int)
diff --git a/pypy/module/select/test/test_kqueue.py b/pypy/module/select/test/test_kqueue.py
--- a/pypy/module/select/test/test_kqueue.py
+++ b/pypy/module/select/test/test_kqueue.py
@@ -186,3 +186,10 @@
         a.close()
         b.close()
         kq.close()
+
+    def test_non_inheritable(self):
+        import select, posix
+
+        kq = select.kqueue()
+        assert posix.get_inheritable(kq.fileno()) == False
+        kq.close()

From pypy.commits at gmail.com  Fri Aug 26 10:32:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:32:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: Catch OSErrors
Message-ID: <57c052ea.262ec20a.3aef8.909a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86568:6fc5d0557a84
Date: 2016-08-26 16:31 +0200
http://bitbucket.org/pypy/pypy/changeset/6fc5d0557a84/

Log:	Catch OSErrors

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -896,11 +896,17 @@
 
 @unwrap_spec(fd=c_int)
 def get_inheritable(space, fd):
-    return space.wrap(rposix.get_inheritable(fd))
+    try:
+        return space.wrap(rposix.get_inheritable(fd))
+    except OSError as e:
+        raise wrap_oserror(space, e)
 
 @unwrap_spec(fd=c_int, inheritable=int)
 def set_inheritable(space, fd, inheritable):
-    rposix.set_inheritable(fd, inheritable)
+    try:
+        rposix.set_inheritable(fd, inheritable)
+    except OSError as e:
+        raise wrap_oserror(space, e)
 
 _pipe_inhcache = rposix.SetNonInheritableCache()
 

From pypy.commits at gmail.com  Fri Aug 26 10:33:23 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 26 Aug 2016 07:33:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: work in progress for
 memoryview.cast, there is still one issue,
 memoryview modifies the format field othe underlying Py_buffer struct
Message-ID: <57c05333.e16ec20a.9378f.138f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86569:5ef2fe0ae4aa
Date: 2016-08-26 16:32 +0200
http://bitbucket.org/pypy/pypy/changeset/5ef2fe0ae4aa/

Log:	work in progress for memoryview.cast, there is still one issue,
	memoryview modifies the format field othe underlying Py_buffer
	struct

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1439,12 +1439,6 @@
     BUF_FULL_RO = BUF_INDIRECT | BUF_FORMAT
     BUF_FULL    = BUF_INDIRECT | BUF_FORMAT | BUF_WRITABLE
 
-    MEMORYVIEW_MAX_DIM = 64
-    MEMORYVIEW_C        = 0x0002
-    MEMORYVIEW_FORTRAN  = 0x0004
-    MEMORYVIEW_SCLAR    = 0x0008
-    MEMORYVIEW_PIL      = 0x0010
-
     def check_buf_flags(self, flags, readonly):
         if readonly and flags & self.BUF_WRITABLE == self.BUF_WRITABLE:
             raise oefmt(self.w_BufferError, "Object is not writable.")
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -11,6 +11,14 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
 from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
+from rpython.rlib.unroll import unrolling_iterable
+
+MEMORYVIEW_MAX_DIM = 64
+MEMORYVIEW_SCALAR   = 0x0001
+MEMORYVIEW_C        = 0x0002
+MEMORYVIEW_FORTRAN  = 0x0004
+MEMORYVIEW_SCALAR   = 0x0008
+MEMORYVIEW_PIL      = 0x0010
 
 
 class W_MemoryView(W_Root):
@@ -18,12 +26,24 @@
     an interp-level buffer.
     """
 
-    def __init__(self, buf, format='B', itemsize=1):
+    def __init__(self, buf, format=None, itemsize=1):
         assert isinstance(buf, Buffer)
         self.buf = buf
         self._hash = -1
         self.format = format
         self.itemsize = itemsize
+        self.flags = 0
+        self._init_flags()
+
+    def getformat(self):
+        # memoryview needs to modify the field 'format', to prevent the modification
+        # of the buffer, we save the new format here!
+        if self.format is None:
+            return self.buf.getformat()
+        return self.format
+
+    def setformat(self, value):
+        self.format = value
 
     def buffer_w_ex(self, space, flags):
         self._check_released(space)
@@ -154,11 +174,11 @@
             # TODO: this probably isn't very fast
             buf = SubBuffer(self.buf, start, self.itemsize)
             fmtiter = UnpackFormatIterator(space, buf)
-            fmtiter.interpret(self.format)
+            fmtiter.interpret(self.getformat())
             return fmtiter.result_w[0]
         elif step == 1:
             buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf, self.format, self.itemsize)
+            return W_MemoryView(buf, self.getformat(), self.itemsize)
         else:
             buf = SubBuffer(self.buf, start, size)
             return W_MemoryView(buf)
@@ -193,7 +213,7 @@
             except StructError as e:
                 raise oefmt(space.w_TypeError,
                             "memoryview: invalid type for format '%s'",
-                            self.format)
+                            self.getformat())
             self.buf.setslice(start, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
@@ -210,7 +230,7 @@
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getformat())
+        return space.wrap(self.getformat())
 
     def w_get_itemsize(self, space):
         self._check_released(space)
@@ -318,7 +338,6 @@
         return False
 
     def descr_cast(self, space, w_format, w_shape=None):
-        # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
         self._check_released(space)
 
         if not space.isinstance_w(w_format, space.w_unicode):
@@ -326,15 +345,15 @@
                     space.wrap("memoryview: format argument must be a string"))
 
         fmt = space.str_w(w_format)
-        view = self.buf
+        buf = self.buf
         ndim = 1
 
-        if not memory_view_c_contiguous(space, view.flags):
+        if not memory_view_c_contiguous(space, self.flags):
             raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: casts are restricted" \
                                " to C-contiguous views"))
 
-        if (w_shape or view.getndim() != 1) and self._zero_in_shape():
+        if (w_shape or buf.getndim() != 1) and self._zero_in_shape():
             raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: cannot casts view with" \
                                " zeros in shape or strides"))
@@ -352,60 +371,89 @@
                 raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: cast must be 1D -> ND or ND -> 1D"))
 
+        mv = W_MemoryView(buf, fmt, itemsize)
+        origfmt = mv.getformat()
+        mv._cast_to_1D(space, origfmt, fmt, itemsize)
+        if w_shape:
             shape = [space.int_w(w_obj) for w_obj in w_shape.fixedview_unroll()]
-            return W_MemoryView(Buffer.cast_to(buf, itemsize, shape))
-
-        return W_MemoryView(Buffer.cast_to(buf, itemsize, None))
+            mv._cast_to_ND(space, shape, dim)
+        return mv
 
     def _init_flags(self):
-        # TODO move to buffer.py
-        view = self.buf
-        ndim = view.ndim
+        buf = self.buf
+        ndim = buf.ndim
         flags = 0
         if ndim == 0:
-            flags |= space.MEMORYVIEW_SCALAR | space.MEMORYVIEW_C | space.MEMORYVIEW_FORTRAN
+            flags |= MEMORYVIEW_SCALAR | MEMORYVIEW_C | MEMORYVIEW_FORTRAN
         if ndim == 1:
-            if view.shape[0] == 1 and view.strides[0] == view.itemsize:
-                flags |= space.MEMORYVIEW_C | space.MEMORYVIEW_SCALAR
-        if view.is_contiguous('C'):
-            flags |= space.MEMORYVIEW_C
-        elif view.is_contiguous('F'):
-            flags |= space.MEMORYVIEW_SCALAR
+            shape = buf.getshape()
+            strides = buf.getstrides()
+            if len(shape) > 0 and shape[0] == 1 and \
+               len(strides) > 0 and strides[0] == buf.getitemsize():
+                flags |= MEMORYVIEW_C | MEMORYVIEW_SCALAR
+        if buf.is_contiguous('C'):
+            flags |= MEMORYVIEW_C
+        elif buf.is_contiguous('F'):
+            flags |= MEMORYVIEW_FORTRAN
 
         # XXX missing suboffsets
 
-        view.flags = flags
+        self.flags = flags
 
-    def _cast_to_1D(self, space, fmt):
-        itemsize = self.get_native_fmtchar(fmt)
+    def _cast_to_1D(self, space, origfmt, fmt, itemsize):
         buf = self.buf
         if itemsize < 0:
-            raise OperationError(space.w_ValueError, "memoryview: destination" \
+            raise oefmt(space.w_ValueError, "memoryview: destination" \
                     " format must be a native single character format prefixed" \
                     " with an optional '@'")
 
-        buffmt = buf.getformat()
-        if self.get_native_fmtchar(buffmt) < 0 or \
-           (not is_byte_format(fmt) and not is_byte_format(buffmt)):
-            raise OperationError(space.w_TypeError,
+        if self.get_native_fmtchar(origfmt) < 0 or \
+           (not is_byte_format(fmt) and not is_byte_format(origfmt)):
+            raise oefmt(space.w_TypeError,
                     "memoryview: cannot cast between" \
                     " two non-byte formats")
 
         if buf.getlength() % itemsize != 0:
-            raise OperationError(space.w_TypeError,
+            raise oefmt(space.w_TypeError,
                     "memoryview: length is not a multiple of itemsize")
 
-        buf.format = get_native_fmtstr(fmt)
-        if not buffmt:
-            raise OperationError(space.w_RuntimeError,
+        newfmt = self.get_native_fmtstr(fmt)
+        if not newfmt:
+            raise oefmt(space.w_RuntimeError,
                     "memoryview: internal error")
-        buf.itemsize = itemsize
-        buf.ndim = 1
-        buf.shape[0] = buf.length / buf.itemsize
-        buf.srides[0] = buf.itemsize
+        self.setformat(newfmt)
+        self.itemsize = itemsize
+        self.ndim = 1
+        self.shape = [buf.getlength() / buf.getitemsize()]
+        self.srides = [buf.getitemsize()]
         # XX suboffsets
 
-        mv._init_flags()
+        self._init_flags()
+
+    def get_native_fmtstr(self, fmt):
+        lenfmt = len(fmt)
+        nat = False
+        if lenfmt == 0:
+            return None
+        elif lenfmt == 1:
+            format = fmt[0] # fine!
+        elif lenfmt == 2:
+            if fmt[0] == '@':
+                nat = True
+                format = fmt[1]
+            else:
+                return None
+        else:
+            return None
+
+        chars = ['c','b','B','h','H','i','I','l','L','q',
+                 'Q','n','N','f','d','?','P']
+        for c in unrolling_iterable(chars):
+            if c == format:
+                if nat: return '@'+c
+                else: return c
+
+        return None
 
     def _cast_to_ND(self, space, shape, ndim):
         pass
@@ -419,7 +467,7 @@
     return char == 'b' or char == 'B' or char == 'c'
 
 def memory_view_c_contiguous(space, flags):
-    return flags & (space.BUF_CONTIG_RO|space.MEMORYVIEW_C) != 0
+    return flags & (space.BUF_CONTIG_RO|MEMORYVIEW_C) != 0
 
 W_MemoryView.typedef = TypeDef(
     "memoryview",
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -177,11 +177,10 @@
     def __init__(self, space, w_arr, w_dim, w_fmt, \
                  w_itemsize, w_strides, w_shape):
         self.space = space
-        self.flags = space.MEMORYVIEW_C
         self.w_arr = w_arr
         self.arr = []
         self.ndim = space.int_w(w_dim)
-        self.fmt = space.str_w(w_fmt)
+        self.format = space.str_w(w_fmt)
         self.itemsize = space.int_w(w_itemsize)
         self.strides = []
         for w_i in w_strides.getitems_unroll():
@@ -204,6 +203,8 @@
 
     def getslice(self, start, stop, step, size):
         items = []
+        if size == 0:
+            return ''
         bytecount = (stop - start)
         # data is stores as list of ints, thus this gets around the
         # issue that one cannot advance in bytes
@@ -214,10 +215,10 @@
         return ''.join(items)
 
     def getformat(self):
-        return self.fmt
+        return self.format
 
     def getitem(self, index):
-        return struct.pack(self.fmt, self.data[index])
+        return struct.pack(self.format, self.data[index])
 
     def getlength(self):
         return len(self.data) * self.itemsize
@@ -234,6 +235,9 @@
     def getshape(self):
         return self.shape
 
+    def is_contiguous(self, format):
+        return format == 'C'
+
 class W_MockArray(W_Root):
     def __init__(self, w_list, w_dim, w_fmt, w_size, w_strides, w_shape):
         self.w_list = w_list
@@ -295,9 +299,29 @@
         assert view[0,0,0] == 1
         assert view[-1,2,0] == 6
 
+    def test_cast_non_byte(self):
+        empty = self.MockArray([], dim=1, fmt='i', size=4, strides=[1], shape=[1])
+        view = memoryview(empty)
+        try:
+            view.cast('l')
+            assert False, "i -> l not possible. buffer must be byte format"
+        except TypeError:
+            pass
+
     def test_cast_empty(self):
-        empty = self.MockArray([], dim=1, fmt='i', size=4, strides=[1], shape=[1])
+        empty = self.MockArray([], dim=1, fmt='b', size=1, strides=[1], shape=[1])
         view = memoryview(empty)
         cview = view.cast('i')
         assert cview.tobytes() == b''
+        assert cview.tolist() == []
+        assert view.format == 'b'
+        assert cview.format == 'i'
+        #
+        assert cview.cast('i').cast('b').cast('i').tolist() == []
+        #
+        try:
+            cview = view.cast('i')
+            assert False, "cannot cast between two non byte formats!"
+        except TypeError:
+            pass
 

From pypy.commits at gmail.com  Fri Aug 26 10:33:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:33:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: select.devpoll()... is not
 implemented so far, but write it down anyway
Message-ID: <57c05357.09afc20a.8df27.8b58@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86570:0a3d2b164e27
Date: 2016-08-26 16:33 +0200
http://bitbucket.org/pypy/pypy/changeset/0a3d2b164e27/

Log:	select.devpoll()... is not implemented so far, but write it down
	anyway

diff --git a/pypy/module/select/test/test_devpoll.py b/pypy/module/select/test/test_devpoll.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/select/test/test_devpoll.py
@@ -0,0 +1,4 @@
+# XXX
+
+# devpoll is not implemented, but if we do, make sure we test for
+# non-inheritable file descriptors

From pypy.commits at gmail.com  Fri Aug 26 10:44:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 07:44:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: Minor leak when the C call socketpair()
 fails
Message-ID: <57c055ba.11051c0a.b5f3a.c877@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86571:95c08fa82fe1
Date: 2016-08-26 16:43 +0200
http://bitbucket.org/pypy/pypy/changeset/95c08fa82fe1/

Log:	Minor leak when the C call socketpair() fails

diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1099,12 +1099,14 @@
         AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
         """
         result = lltype.malloc(_c.socketpair_t, 2, flavor='raw')
-        res = _c.socketpair(family, type, proto, result)
-        if res < 0:
-            raise last_error()
-        fd0 = rffi.cast(lltype.Signed, result[0])
-        fd1 = rffi.cast(lltype.Signed, result[1])
-        lltype.free(result, flavor='raw')
+        try:
+            res = _c.socketpair(family, type, proto, result)
+            if res < 0:
+                raise last_error()
+            fd0 = rffi.cast(lltype.Signed, result[0])
+            fd1 = rffi.cast(lltype.Signed, result[1])
+        finally:
+            lltype.free(result, flavor='raw')
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 

From pypy.commits at gmail.com  Fri Aug 26 11:28:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 08:28:48 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: update
Message-ID: <57c06030.a717c20a.2f7c9.ad96@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5683:348534fd42b4
Date: 2016-08-26 17:28 +0200
http://bitbucket.org/pypy/extradoc/changeset/348534fd42b4/

Log:	update

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -22,3 +22,9 @@
 * os.scandir() direntry objects should not have stat() called from two
   threads concurrently.  It will make two stat objects and leak one of
   them.
+
+* (not a crasher) on modern Linux: if the first call in the process to
+  socketpair() ends in a EINVAL, then cpython will (possibly wrongly)
+  assume it was caused by SOCK_CLOEXEC and not use SOCK_CLOEXEC at all
+  in the future
+

From pypy.commits at gmail.com  Fri Aug 26 11:28:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 08:28:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: Add
 rsocket.socketpair(inheritable=..). Fix _socket.socketpair() at
Message-ID: <57c06038.05d71c0a.5de62.e06f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86572:f44ca4fc29df
Date: 2016-08-26 17:28 +0200
http://bitbucket.org/pypy/pypy/changeset/f44ca4fc29df/

Log:	Add rsocket.socketpair(inheritable=..). Fix _socket.socketpair() at
	app-level.

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -170,7 +170,8 @@
     AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
     """
     try:
-        sock1, sock2 = rsocket.socketpair(family, type, proto)
+        sock1, sock2 = rsocket.socketpair(family, type, proto,
+                                          inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
     return space.newtuple([
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -648,6 +648,16 @@
         assert len(w) == 1, [str(warning) for warning in w]
         assert r in str(w[0])
 
+    def test_socketpair_non_inheritable(self):
+        import _socket, posix
+        if not hasattr(_socket, 'socketpair'):
+            skip("no socketpair")
+        s1, s2 = _socket.socketpair()
+        assert posix.get_inheritable(s1.fileno()) is False
+        assert posix.get_inheritable(s2.fileno()) is False
+        s1.close()
+        s2.close()
+
 
 class AppTestNetlink:
     def setup_class(cls):
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -176,6 +176,7 @@
 
 
 SOCK_DGRAM SOCK_RAW SOCK_RDM SOCK_SEQPACKET SOCK_STREAM
+SOCK_CLOEXEC
 
 SOL_SOCKET SOL_IPX SOL_AX25 SOL_ATALK SOL_NETROM SOL_ROSE
 
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -11,7 +11,7 @@
 from rpython.rlib import _rsocket_rffi as _c, jit, rgc
 from rpython.rlib.objectmodel import instantiate, keepalive_until_here
 from rpython.rlib.rarithmetic import intmask, r_uint
-from rpython.rlib import rthread
+from rpython.rlib import rthread, rposix
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rffi import sizeof, offsetof
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -1032,6 +1032,12 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
+def sock_set_inheritable(fd, inheritable):
+    try:
+        rposix.set_inheritable(fd, inheritable)
+    except OSError as e:
+        raise CSocketError(e.errno)
+
 class SocketError(Exception):
     applevelerrcls = 'error'
     def __init__(self):
@@ -1090,7 +1096,7 @@
 
 if hasattr(_c, 'socketpair'):
     def socketpair(family=socketpair_default_family, type=SOCK_STREAM, proto=0,
-                   SocketClass=RSocket):
+                   SocketClass=RSocket, inheritable=True):
         """socketpair([family[, type[, proto]]]) -> (socket object, socket object)
 
         Create a pair of socket objects from the sockets returned by the platform
@@ -1099,12 +1105,37 @@
         AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
         """
         result = lltype.malloc(_c.socketpair_t, 2, flavor='raw')
-        res = _c.socketpair(family, type, proto, result)
-        if res < 0:
-            raise last_error()
-        fd0 = rffi.cast(lltype.Signed, result[0])
-        fd1 = rffi.cast(lltype.Signed, result[1])
-        lltype.free(result, flavor='raw')
+        try:
+            res = -1
+            remove_inheritable = not inheritable
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socketpair() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                res = _c.socketpair(family, type | SOCK_CLOEXEC,
+                                    proto, result)
+                if res < 0:
+                    if _c.geterrno() == errno.EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise last_error()
+                else:
+                    remove_inheritable = False
+            #
+            if res < 0:
+                res = _c.socketpair(family, type, proto, result)
+                if res < 0:
+                    raise last_error()
+            fd0 = rffi.cast(lltype.Signed, result[0])
+            fd1 = rffi.cast(lltype.Signed, result[1])
+        finally:
+            lltype.free(result, flavor='raw')
+        if remove_inheritable:
+            sock_set_inheritable(fd0, False)
+            sock_set_inheritable(fd1, False)
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -119,6 +119,16 @@
     s1.close()
     s2.close()
 
+def test_socketpair_inheritable():
+    if sys.platform == "win32":
+        py.test.skip('No socketpair on Windows')
+    for inh in [False, True]:
+        s1, s2 = socketpair(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        assert rposix.get_inheritable(s2.fd) == inh
+        s1.close()
+        s2.close()
+
 def test_socketpair_recvinto_1():
     class Buffer:
         def setslice(self, start, string):

From pypy.commits at gmail.com  Fri Aug 26 11:49:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 08:49:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: socket.socket()
Message-ID: <57c064f5.09afc20a.8df27.a6bd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86573:b918e61bdf64
Date: 2016-08-26 17:45 +0200
http://bitbucket.org/pypy/pypy/changeset/b918e61bdf64/

Log:	socket.socket()

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -177,7 +177,7 @@
                 sock = RSocket(family, type, proto,
                                fd=space.c_filedescriptor_w(w_fileno))
             else:
-                sock = RSocket(family, type, proto)
+                sock = RSocket(family, type, proto, inheritable=False)
             W_Socket.__init__(self, space, sock)
         except SocketError as e:
             raise converted_error(space, e)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -648,6 +648,12 @@
         assert len(w) == 1, [str(warning) for warning in w]
         assert r in str(w[0])
 
+    def test_socket_non_inheritable(self):
+        import _socket, posix
+        s1 = _socket.socket()
+        assert posix.get_inheritable(s1.fileno()) is False
+        s1.close()
+
     def test_socketpair_non_inheritable(self):
         import _socket, posix
         if not hasattr(_socket, 'socketpair'):
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -522,12 +522,28 @@
     timeout = -1.0
 
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0,
-                 fd=_c.INVALID_SOCKET):
+                 fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            fd = _c.socket(family, type, proto)
-        if _c.invalid_socket(fd):
-            raise self.error_handler()
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socket() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                fd = _c.socket(family, type | SOCK_CLOEXEC, proto)
+                if fd < 0:
+                    if _c.geterrno() == errno.EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise self.error_handler()
+            if _c.invalid_socket(fd):
+                fd = _c.socket(family, type, proto)
+                if _c.invalid_socket(fd):
+                    raise self.error_handler()
+                if not inheritable:
+                    sock_set_inheritable(fd, False)
         # PLAT RISCOS
         self.fd = fd
         self.family = family
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -388,6 +388,12 @@
     s1.close()
     s2.close()
 
+def test_inheritable():
+    for inh in [False, True]:
+        s1 = RSocket(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        s1.close()
+
 def test_getaddrinfo_http():
     lst = getaddrinfo('localhost', 'http')
     assert isinstance(lst, list)

From pypy.commits at gmail.com  Fri Aug 26 11:49:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 08:49:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: add a test (passing)
Message-ID: <57c064f7.497bc20a.5c897.a22b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86574:6aa903a2fc65
Date: 2016-08-26 17:48 +0200
http://bitbucket.org/pypy/pypy/changeset/6aa903a2fc65/

Log:	add a test (passing)

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -648,6 +648,10 @@
         assert len(w) == 1, [str(warning) for warning in w]
         assert r in str(w[0])
 
+    def test_invalid_fd(self):
+        import _socket
+        raises(ValueError, _socket.socket, fileno=-1)
+
     def test_socket_non_inheritable(self):
         import _socket, posix
         s1 = _socket.socket()

From pypy.commits at gmail.com  Fri Aug 26 13:02:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 10:02:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: Cache the ENOSYS error from
 pipe2
Message-ID: <57c0762b.a719c20a.7a270.c3e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86575:8e165943e788
Date: 2016-08-26 18:49 +0200
http://bitbucket.org/pypy/pypy/changeset/8e165943e788/

Log:	Cache the ENOSYS error from pipe2

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1170,9 +1170,10 @@
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
-            if HAVE_PIPE2:
+            if HAVE_PIPE2 and _pipe2_syscall.attempt_syscall():
                 res = c_pipe2(filedes, flags)
-                if widen(res) != 0 and get_saved_errno() == errno.ENOSYS:
+                _pipe2_syscall.update()
+                if res < 0 and get_saved_errno() == errno.ENOSYS:
                     res = c_pipe(filedes)
             else:
                 res = c_pipe(filedes)      # 'flags' ignored
@@ -2267,3 +2268,17 @@
         if self.cached_inheritable == 1:
             # 'fd' is inheritable; we must manually turn it off
             set_inheritable(fd, False)
+
+class ENoSysCache(object):
+    """Cache whether a system call returns ENOSYS or not."""
+    _immutable_fields_ = ['cached_nosys?']
+    cached_nosys = -1      # -1 = don't know; 0 = no; 1 = yes, getting ENOSYS
+
+    def attempt_syscall(self):
+        return self.cached_nosys != 1
+
+    def update(self):
+        if self.cached_nosys == -1:
+            self.cached_nosys = (get_saved_errno() == errno.ENOSYS)
+
+_pipe2_syscall = ENoSysCache()

From pypy.commits at gmail.com  Fri Aug 26 13:02:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 10:02:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: socket.accept()
Message-ID: <57c0762d.85c11c0a.cb5e1.fc2c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86576:e7787d31798f
Date: 2016-08-26 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/e7787d31798f/

Log:	socket.accept()

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -228,7 +228,7 @@
         For IP sockets, the address info is a pair (hostaddr, port).
         """
         try:
-            fd, addr = self.sock.accept()
+            fd, addr = self.sock.accept(inheritable=False)
             return space.newtuple([space.wrap(fd),
                                    addr_as_object(addr, fd, space)])
         except SocketError as e:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -846,6 +846,16 @@
         assert cli.family == socket.AF_INET
 
 
+    def test_accept_non_inheritable(self):
+        import _socket, posix
+        cli = _socket.socket()
+        cli.connect(self.serv.getsockname())
+        fileno, addr = self.serv._accept()
+        assert posix.get_inheritable(fileno) is False
+        posix.close(fileno)
+        cli.close()
+
+
 class AppTestErrno:
     spaceconfig = {'usemodules': ['_socket']}
 
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -320,6 +320,8 @@
                                           [('p_proto', rffi.INT),
                                            ])
 
+CConfig.HAVE_ACCEPT4 = platform.Has('accept4')
+
 if _POSIX:
     CConfig.nfds_t = platform.SimpleType('nfds_t')
     CConfig.pollfd = platform.Struct('struct pollfd',
@@ -542,6 +544,11 @@
 socketaccept = external('accept', [socketfd_type, sockaddr_ptr,
                                    socklen_t_ptr], socketfd_type,
                         save_err=SAVE_ERR)
+if cConfig.HAVE_ACCEPT4:
+    socketaccept4 = external('accept4', [socketfd_type, sockaddr_ptr,
+                                         socklen_t_ptr, rffi.INT],
+                                        socketfd_type,
+                             save_err=SAVE_ERR)
 socketbind = external('bind', [socketfd_type, sockaddr_ptr, socklen_t],
                               rffi.INT, save_err=SAVE_ERR)
 socketlisten = external('listen', [socketfd_type, rffi.INT], rffi.INT,
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1172,8 +1172,7 @@
         try:
             if HAVE_PIPE2 and _pipe2_syscall.attempt_syscall():
                 res = c_pipe2(filedes, flags)
-                _pipe2_syscall.update()
-                if res < 0 and get_saved_errno() == errno.ENOSYS:
+                if _pipe2_syscall.fallback(res):
                     res = c_pipe(filedes)
             else:
                 res = c_pipe(filedes)      # 'flags' ignored
@@ -2269,6 +2268,9 @@
             # 'fd' is inheritable; we must manually turn it off
             set_inheritable(fd, False)
 
+    def _cleanup_(self):
+        self.cached_inheritable = -1
+
 class ENoSysCache(object):
     """Cache whether a system call returns ENOSYS or not."""
     _immutable_fields_ = ['cached_nosys?']
@@ -2277,8 +2279,14 @@
     def attempt_syscall(self):
         return self.cached_nosys != 1
 
-    def update(self):
-        if self.cached_nosys == -1:
-            self.cached_nosys = (get_saved_errno() == errno.ENOSYS)
+    def fallback(self, res):
+        nosys = self.cached_nosys
+        if nosys == -1:
+            nosys = (res < 0 and get_saved_errno() == errno.ENOSYS)
+            self.cached_nosys = nosys
+        return nosys
+
+    def _cleanup_(self):
+        self.cached_nosys = -1
 
 _pipe2_syscall = ENoSysCache()
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -646,20 +646,33 @@
         return addr, addr.addr_p, addrlen_p
 
     @jit.dont_look_inside
-    def accept(self):
+    def accept(self, inheritable=True):
         """Wait for an incoming connection.
         Return (new socket fd, client address)."""
         if self._select(False) == 1:
             raise SocketTimeout
         address, addr_p, addrlen_p = self._addrbuf()
         try:
-            newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+            remove_inheritable = not inheritable
+            if (not inheritable and SOCK_CLOEXEC is not None
+                    and hasattr(_c, 'socketaccept4') and
+                    _accept4_syscall.attempt_syscall()):
+                newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
+                                         SOCK_CLOEXEC)
+                if _accept4_syscall.fallback(newfd):
+                    newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+                else:
+                    remove_inheritable = False
+            else:
+                newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
             addrlen = addrlen_p[0]
         finally:
             lltype.free(addrlen_p, flavor='raw')
             address.unlock()
         if _c.invalid_socket(newfd):
             raise self.error_handler()
+        if remove_inheritable:
+            sock_set_inheritable(newfd, False)
         address.addrlen = rffi.cast(lltype.Signed, addrlen)
         return (newfd, address)
 
@@ -1452,3 +1465,5 @@
     if timeout < 0.0:
         timeout = -1.0
     defaults.timeout = timeout
+
+_accept4_syscall = rposix.ENoSysCache()

From pypy.commits at gmail.com  Fri Aug 26 13:10:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 10:10:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: test and fix for _socket.dup()
Message-ID: <57c0781f.54bc1c0a.66935.ff49@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86577:4c74988264fd
Date: 2016-08-26 19:10 +0200
http://bitbucket.org/pypy/pypy/changeset/4c74988264fd/

Log:	test and fix for _socket.dup()

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -142,7 +142,10 @@
 
 @unwrap_spec(fd=int)
 def dup(space, fd):
-    newfd = rsocket.dup(fd)
+    try:
+        newfd = rsocket.dup(fd)
+    except SocketError as e:
+        raise converted_error(space, e)
     return space.wrap(newfd)
 
 @unwrap_spec(fd=int, family=int, type=int, proto=int)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -552,6 +552,10 @@
         fd = socket.dup(s.fileno())
         assert s.fileno() != fd
 
+    def test_dup_error(self):
+        import _socket
+        raises(_socket.error, _socket.dup, 123456)
+
     def test_buffer(self):
         # Test that send/sendall/sendto accept a buffer as arg
         import _socket, os
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1121,14 +1121,14 @@
             return result
 else:
     def dup(fd):
-        return _c.dup(fd)
-
-    def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
-        # Dup the fd so it and the socket can be closed independently
         fd = _c.dup(fd)
         if fd < 0:
             raise last_error()
-        return make_socket(fd, family, type, proto, SocketClass)
+        return fd
+
+def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
+    # Dup the fd so it and the socket can be closed independently
+    return make_socket(dup(fd), family, type, proto, SocketClass)
 
 def getdefaulttimeout():
     return defaults.timeout

From pypy.commits at gmail.com  Fri Aug 26 13:12:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 10:12:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c07863.d8011c0a.577cb.0bae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86578:2eee1909c4d1
Date: 2016-08-26 19:11 +0200
http://bitbucket.org/pypy/pypy/changeset/2eee1909c4d1/

Log:	hg merge py3k

diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py
deleted file mode 100644
--- a/pypy/module/_file/readinto.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import sys, errno
-from rpython.rlib import rposix
-from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._file.interp_file import is_wouldblock_error, signal_checker
-
-_WIN32 = sys.platform.startswith('win')
-UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
-
-os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read',
-                          [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
-                          rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO)
-
-
-def direct_readinto(self, w_rwbuffer):
-    rwbuffer = self.space.writebuf_w(w_rwbuffer)
-    stream = self.getstream()
-    size = rwbuffer.getlength()
-    target_address = lltype.nullptr(rffi.CCHARP.TO)
-    fd = -1
-    target_pos = 0
-
-    if size > 64:
-        try:
-            target_address = rwbuffer.get_raw_address()
-        except ValueError:
-            pass
-        else:
-            fd = stream.try_to_find_file_descriptor()
-
-    if fd < 0 or not target_address:
-        # fall-back
-        MAX_PART = 1024 * 1024    # 1 MB
-        while size > MAX_PART:
-            data = self.direct_read(MAX_PART)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-            if len(data) != MAX_PART:
-                break
-        else:
-            data = self.direct_read(size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-
-    else:
-        # optimized case: reading more than 64 bytes into a rwbuffer
-        # with a valid raw address
-        self.check_readable()
-
-        # first "read" the part that is already sitting in buffers, if any
-        initial_size = min(size, stream.count_buffered_bytes())
-        if initial_size > 0:
-            data = stream.read(initial_size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-
-        # then call os_read() to get the rest
-        if size > 0:
-            stream.flush()
-            while True:
-                got = os_read(fd, rffi.ptradd(target_address, target_pos), size)
-                got = rffi.cast(lltype.Signed, got)
-                if got > 0:
-                    target_pos += got
-                    size -= got
-                    if size <= 0:
-                        break
-                elif got == 0:
-                    break
-                else:
-                    err = rposix.get_saved_errno()
-                    if err == errno.EINTR:
-                        signal_checker(self.space)()
-                        continue
-                    if is_wouldblock_error(err) and target_pos > 0:
-                        break
-                    raise OSError(err, "read error")
-            keepalive_until_here(rwbuffer)
-
-    return self.space.wrap(target_pos)
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -142,7 +142,10 @@
 
 @unwrap_spec(fd=int)
 def dup(space, fd):
-    newfd = rsocket.dup(fd)
+    try:
+        newfd = rsocket.dup(fd)
+    except SocketError as e:
+        raise converted_error(space, e)
     return space.wrap(newfd)
 
 @unwrap_spec(fd=int, family=int, type=int, proto=int)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -552,6 +552,10 @@
         fd = socket.dup(s.fileno())
         assert s.fileno() != fd
 
+    def test_dup_error(self):
+        import _socket
+        raises(_socket.error, _socket.dup, 123456)
+
     def test_buffer(self):
         # Test that send/sendall/sendto accept a buffer as arg
         import _socket, os
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -19,6 +19,7 @@
         'statvfs_result': 'app_posix.statvfs_result',
         'uname_result': 'app_posix.uname_result',
         'urandom': 'app_posix.urandom',
+        'terminal_size': 'app_posix.terminal_size'
     }
     if os.name == 'nt':
         del appleveldefs['urandom'] # at interp on win32
@@ -74,6 +75,7 @@
         'abort': 'interp_posix.abort',
         'urandom': 'interp_posix.urandom',
         'device_encoding': 'interp_posix.device_encoding',
+        'get_terminal_size': 'interp_posix.get_terminal_size'
 
         'scandir': 'interp_scandir.scandir',
     }
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -92,6 +92,12 @@
     version  = structseqfield(3, "operating system version")
     machine  = structseqfield(4, "hardware identifier")
 
+class terminal_size(metaclass=structseqtype):
+
+    name = "os.terminal_size"
+
+    columns  = structseqfield(0, "width of the terminal window in characters")
+    lines = structseqfield(1, "height of the terminal window in characters")
 
 if osname == 'posix':
     # POSIX: we want to check the file descriptor when fdopen() is called,
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -8,16 +8,18 @@
     # some Pythons don't have errno.ENOTSUP
     ENOTSUP = 0
 
-from rpython.rlib import rposix, rposix_stat
+from rpython.rlib import rposix, rposix_stat, rfile
 from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint
+from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint, r_int
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rtyper.lltypesystem import lltype
 from rpython.tool.sourcetools import func_with_new_name
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, Unwrapper
 from pypy.interpreter.error import (
-    OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror)
+    OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror,
+    exception_from_saved_errno)
 from pypy.interpreter.executioncontext import ExecutionContext
 
 
@@ -2142,3 +2144,50 @@
         have_functions.append("HAVE_%s" % name)
 if _WIN32:
     have_functions.append("HAVE_MS_WINDOWS")
+   
+def get_terminal_size(space, w_fd=None):
+    if w_fd is None:
+        fd = rfile.RFile(rfile.c_stdout(), close2=(None, None)).fileno()
+    else:
+        if not space.isinstance_w(w_fd, space.w_int):
+            raise oefmt(space.w_TypeError,
+                        "an integer is required, got %T", w_fd)
+        else:
+            fd = space.c_int_w(w_fd)
+
+    if _WIN32:
+        if fd == 0:
+            handle_id = rwin32.STD_INPUT_HANDLE
+        elif fd == 1:
+            handle_id = rwin32.STD_OUTPUT_HANDLE
+        elif fd == 2:
+            handle_id = rwin32.STD_ERROR_HANDLE
+        else:
+            raise oefmt(space.w_ValueError, "bad file descriptor")
+
+        handle = rwin32.GetStdHandle(handle_id)
+
+        if handle == rwin32.NULL_HANDLE:
+            raise oefmt(space.w_OSError, "handle cannot be retrieved")
+        elif handle == rwin32.INVALID_HANDLE_VALUE:
+            raise rwin32.lastSavedWindowsError()
+        with lltype.scoped_alloc(rwin32.CONSOLE_SCREEN_BUFFER_INFO) as buffer_info: 
+            success = rwin32.GetConsoleScreenBufferInfo(handle, buffer_info)
+            if not success:
+                raise rwin32.lastSavedWindowsError()
+            w_columns = space.wrap(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
+            w_lines = space.wrap(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
+    else:
+        with lltype.scoped_alloc(rposix.WINSIZE) as winsize: 
+            failed = rposix.c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
+            if failed:
+                raise exception_from_saved_errno(space, space.w_OSError)
+
+            w_columns = space.wrap(r_uint(winsize.c_ws_col))
+            w_lines = space.wrap(r_uint(winsize.c_ws_row))
+
+    w_tuple = space.newtuple([w_columns, w_lines])
+    w_terminal_size = space.getattr(space.getbuiltinmodule(os.name),
+                                    space.wrap('terminal_size'))
+
+    return space.call_function(w_terminal_size, w_tuple)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -139,12 +139,10 @@
         space = self.space
 
         if not space.config.translating:
-            ##from pypy.module.sys.interp_encoding import _getfilesystemencoding
-            ##self.filesystemencoding = _getfilesystemencoding(space)
+            from pypy.module.sys.interp_encoding import _getfilesystemencoding
+            self.filesystemencoding = _getfilesystemencoding(space)
             # XXX the two lines above take a few seconds to run whenever
-            # we initialize the space; for tests, use a simpler version
-            from pypy.module.sys.interp_encoding import base_encoding
-            self.filesystemencoding = base_encoding
+            # we initialize the space
 
     def flush_std_files(self, space):
         w_stdout = space.sys.getdictvalue(space, 'stdout')
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -258,6 +258,7 @@
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
+        TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),
@@ -265,6 +266,12 @@
                            ('tms_cutime', rffi.INT),
                            ('tms_cstime', rffi.INT)])
 
+        WINSIZE = rffi_platform.Struct(
+            'struct winsize', [('ws_row', rffi.USHORT),
+                           ('ws_col', rffi.USHORT),
+                           ('ws_xpixel', rffi.USHORT),
+                           ('ws_ypixel', rffi.USHORT)])
+
     GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
     SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")
 
@@ -628,6 +635,8 @@
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
     c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+    c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 else:
     dirent_config = {}
 
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1121,14 +1121,14 @@
             return result
 else:
     def dup(fd):
-        return _c.dup(fd)
-
-    def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
-        # Dup the fd so it and the socket can be closed independently
         fd = _c.dup(fd)
         if fd < 0:
             raise last_error()
-        return make_socket(fd, family, type, proto, SocketClass)
+        return fd
+
+def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
+    # Dup the fd so it and the socket can be closed independently
+    return make_socket(dup(fd), family, type, proto, SocketClass)
 
 def getdefaulttimeout():
     return defaults.timeout
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -59,6 +59,24 @@
         SYSTEMTIME = rffi_platform.Struct('SYSTEMTIME',
                                           [])
 
+        Struct = rffi_platform.Struct
+        COORD = Struct("COORD",
+                       [("X", rffi.SHORT),
+                        ("Y", rffi.SHORT)])
+
+        SMALL_RECT = Struct("SMALL_RECT",
+                            [("Left", rffi.SHORT),
+                             ("Top", rffi.SHORT),
+                             ("Right", rffi.SHORT),
+                             ("Bottom", rffi.SHORT)])
+
+        CONSOLE_SCREEN_BUFFER_INFO = Struct("CONSOLE_SCREEN_BUFFER_INFO",
+                                            [("dwSize", COORD),
+                                             ("dwCursorPosition", COORD),
+                                             ("wAttributes", WORD.ctype_hint),
+                                             ("srWindow", SMALL_RECT),
+                                             ("dwMaximumWindowSize", COORD)])
+
         OSVERSIONINFOEX = rffi_platform.Struct(
             'OSVERSIONINFOEX',
             [('dwOSVersionInfoSize', rffi.UINT),
@@ -93,7 +111,8 @@
                        PROCESS_VM_WRITE
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
-                       WC_NO_BEST_FIT_CHARS
+                       WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
+                       STD_ERROR_HANDLE
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -444,3 +463,13 @@
         return rffi.cast(lltype.Signed, _GetConsoleOutputCP())
 
     _wenviron_items, _wgetenv, _wputenv = make_env_impls(win32=True)
+
+
+    _GetStdHandle = winexternal(
+        'GetStdHandle', [DWORD], HANDLE)
+
+    def GetStdHandle(handle_id):
+        return _GetStdHandle(handle_id)
+    CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
+    GetConsoleScreenBufferInfo = winexternal(
+        "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)

From pypy.commits at gmail.com  Fri Aug 26 13:12:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 10:12:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: hg merge py3.5
Message-ID: <57c07865.c62f1c0a.9f43d.f625@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86579:04ab431f72ec
Date: 2016-08-26 19:11 +0200
http://bitbucket.org/pypy/pypy/changeset/04ab431f72ec/

Log:	hg merge py3.5

diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py
deleted file mode 100644
--- a/pypy/module/_file/readinto.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import sys, errno
-from rpython.rlib import rposix
-from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._file.interp_file import is_wouldblock_error, signal_checker
-
-_WIN32 = sys.platform.startswith('win')
-UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
-
-os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read',
-                          [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
-                          rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO)
-
-
-def direct_readinto(self, w_rwbuffer):
-    rwbuffer = self.space.writebuf_w(w_rwbuffer)
-    stream = self.getstream()
-    size = rwbuffer.getlength()
-    target_address = lltype.nullptr(rffi.CCHARP.TO)
-    fd = -1
-    target_pos = 0
-
-    if size > 64:
-        try:
-            target_address = rwbuffer.get_raw_address()
-        except ValueError:
-            pass
-        else:
-            fd = stream.try_to_find_file_descriptor()
-
-    if fd < 0 or not target_address:
-        # fall-back
-        MAX_PART = 1024 * 1024    # 1 MB
-        while size > MAX_PART:
-            data = self.direct_read(MAX_PART)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-            if len(data) != MAX_PART:
-                break
-        else:
-            data = self.direct_read(size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-
-    else:
-        # optimized case: reading more than 64 bytes into a rwbuffer
-        # with a valid raw address
-        self.check_readable()
-
-        # first "read" the part that is already sitting in buffers, if any
-        initial_size = min(size, stream.count_buffered_bytes())
-        if initial_size > 0:
-            data = stream.read(initial_size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-
-        # then call os_read() to get the rest
-        if size > 0:
-            stream.flush()
-            while True:
-                got = os_read(fd, rffi.ptradd(target_address, target_pos), size)
-                got = rffi.cast(lltype.Signed, got)
-                if got > 0:
-                    target_pos += got
-                    size -= got
-                    if size <= 0:
-                        break
-                elif got == 0:
-                    break
-                else:
-                    err = rposix.get_saved_errno()
-                    if err == errno.EINTR:
-                        signal_checker(self.space)()
-                        continue
-                    if is_wouldblock_error(err) and target_pos > 0:
-                        break
-                    raise OSError(err, "read error")
-            keepalive_until_here(rwbuffer)
-
-    return self.space.wrap(target_pos)
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -142,7 +142,10 @@
 
 @unwrap_spec(fd=int)
 def dup(space, fd):
-    newfd = rsocket.dup(fd)
+    try:
+        newfd = rsocket.dup(fd)
+    except SocketError as e:
+        raise converted_error(space, e)
     return space.wrap(newfd)
 
 @unwrap_spec(fd=int, family=int, type=int, proto=int)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -552,6 +552,10 @@
         fd = socket.dup(s.fileno())
         assert s.fileno() != fd
 
+    def test_dup_error(self):
+        import _socket
+        raises(_socket.error, _socket.dup, 123456)
+
     def test_buffer(self):
         # Test that send/sendall/sendto accept a buffer as arg
         import _socket, os
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -19,6 +19,7 @@
         'statvfs_result': 'app_posix.statvfs_result',
         'uname_result': 'app_posix.uname_result',
         'urandom': 'app_posix.urandom',
+        'terminal_size': 'app_posix.terminal_size'
     }
     if os.name == 'nt':
         del appleveldefs['urandom'] # at interp on win32
@@ -74,6 +75,7 @@
         'abort': 'interp_posix.abort',
         'urandom': 'interp_posix.urandom',
         'device_encoding': 'interp_posix.device_encoding',
+        'get_terminal_size': 'interp_posix.get_terminal_size'
 
         'scandir': 'interp_scandir.scandir',
         'get_inheritable': 'interp_posix.get_inheritable',
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -92,6 +92,12 @@
     version  = structseqfield(3, "operating system version")
     machine  = structseqfield(4, "hardware identifier")
 
+class terminal_size(metaclass=structseqtype):
+
+    name = "os.terminal_size"
+
+    columns  = structseqfield(0, "width of the terminal window in characters")
+    lines = structseqfield(1, "height of the terminal window in characters")
 
 if osname == 'posix':
     # POSIX: we want to check the file descriptor when fdopen() is called,
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -8,16 +8,18 @@
     # some Pythons don't have errno.ENOTSUP
     ENOTSUP = 0
 
-from rpython.rlib import rposix, rposix_stat
+from rpython.rlib import rposix, rposix_stat, rfile
 from rpython.rlib import objectmodel, rurandom
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint
+from rpython.rlib.rarithmetic import r_longlong, intmask, r_uint, r_int
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rtyper.lltypesystem import lltype
 from rpython.tool.sourcetools import func_with_new_name
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault, Unwrapper
 from pypy.interpreter.error import (
-    OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror)
+    OperationError, oefmt, wrap_oserror, wrap_oserror2, strerror as _strerror,
+    exception_from_saved_errno)
 from pypy.interpreter.executioncontext import ExecutionContext
 
 
@@ -2172,3 +2174,50 @@
         have_functions.append("HAVE_%s" % name)
 if _WIN32:
     have_functions.append("HAVE_MS_WINDOWS")
+   
+def get_terminal_size(space, w_fd=None):
+    if w_fd is None:
+        fd = rfile.RFile(rfile.c_stdout(), close2=(None, None)).fileno()
+    else:
+        if not space.isinstance_w(w_fd, space.w_int):
+            raise oefmt(space.w_TypeError,
+                        "an integer is required, got %T", w_fd)
+        else:
+            fd = space.c_int_w(w_fd)
+
+    if _WIN32:
+        if fd == 0:
+            handle_id = rwin32.STD_INPUT_HANDLE
+        elif fd == 1:
+            handle_id = rwin32.STD_OUTPUT_HANDLE
+        elif fd == 2:
+            handle_id = rwin32.STD_ERROR_HANDLE
+        else:
+            raise oefmt(space.w_ValueError, "bad file descriptor")
+
+        handle = rwin32.GetStdHandle(handle_id)
+
+        if handle == rwin32.NULL_HANDLE:
+            raise oefmt(space.w_OSError, "handle cannot be retrieved")
+        elif handle == rwin32.INVALID_HANDLE_VALUE:
+            raise rwin32.lastSavedWindowsError()
+        with lltype.scoped_alloc(rwin32.CONSOLE_SCREEN_BUFFER_INFO) as buffer_info: 
+            success = rwin32.GetConsoleScreenBufferInfo(handle, buffer_info)
+            if not success:
+                raise rwin32.lastSavedWindowsError()
+            w_columns = space.wrap(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
+            w_lines = space.wrap(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
+    else:
+        with lltype.scoped_alloc(rposix.WINSIZE) as winsize: 
+            failed = rposix.c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
+            if failed:
+                raise exception_from_saved_errno(space, space.w_OSError)
+
+            w_columns = space.wrap(r_uint(winsize.c_ws_col))
+            w_lines = space.wrap(r_uint(winsize.c_ws_row))
+
+    w_tuple = space.newtuple([w_columns, w_lines])
+    w_terminal_size = space.getattr(space.getbuiltinmodule(os.name),
+                                    space.wrap('terminal_size'))
+
+    return space.call_function(w_terminal_size, w_tuple)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -139,12 +139,10 @@
         space = self.space
 
         if not space.config.translating:
-            ##from pypy.module.sys.interp_encoding import _getfilesystemencoding
-            ##self.filesystemencoding = _getfilesystemencoding(space)
+            from pypy.module.sys.interp_encoding import _getfilesystemencoding
+            self.filesystemencoding = _getfilesystemencoding(space)
             # XXX the two lines above take a few seconds to run whenever
-            # we initialize the space; for tests, use a simpler version
-            from pypy.module.sys.interp_encoding import base_encoding
-            self.filesystemencoding = base_encoding
+            # we initialize the space
 
     def flush_std_files(self, space):
         w_stdout = space.sys.getdictvalue(space, 'stdout')
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -258,6 +258,7 @@
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
+        TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),
@@ -265,6 +266,12 @@
                            ('tms_cutime', rffi.INT),
                            ('tms_cstime', rffi.INT)])
 
+        WINSIZE = rffi_platform.Struct(
+            'struct winsize', [('ws_row', rffi.USHORT),
+                           ('ws_col', rffi.USHORT),
+                           ('ws_xpixel', rffi.USHORT),
+                           ('ws_ypixel', rffi.USHORT)])
+
     GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
     SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")
 
@@ -636,6 +643,8 @@
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
     c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+    c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 else:
     dirent_config = {}
 
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1181,14 +1181,14 @@
             return result
 else:
     def dup(fd):
-        return _c.dup(fd)
-
-    def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
-        # Dup the fd so it and the socket can be closed independently
         fd = _c.dup(fd)
         if fd < 0:
             raise last_error()
-        return make_socket(fd, family, type, proto, SocketClass)
+        return fd
+
+def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
+    # Dup the fd so it and the socket can be closed independently
+    return make_socket(dup(fd), family, type, proto, SocketClass)
 
 def getdefaulttimeout():
     return defaults.timeout
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -59,6 +59,24 @@
         SYSTEMTIME = rffi_platform.Struct('SYSTEMTIME',
                                           [])
 
+        Struct = rffi_platform.Struct
+        COORD = Struct("COORD",
+                       [("X", rffi.SHORT),
+                        ("Y", rffi.SHORT)])
+
+        SMALL_RECT = Struct("SMALL_RECT",
+                            [("Left", rffi.SHORT),
+                             ("Top", rffi.SHORT),
+                             ("Right", rffi.SHORT),
+                             ("Bottom", rffi.SHORT)])
+
+        CONSOLE_SCREEN_BUFFER_INFO = Struct("CONSOLE_SCREEN_BUFFER_INFO",
+                                            [("dwSize", COORD),
+                                             ("dwCursorPosition", COORD),
+                                             ("wAttributes", WORD.ctype_hint),
+                                             ("srWindow", SMALL_RECT),
+                                             ("dwMaximumWindowSize", COORD)])
+
         OSVERSIONINFOEX = rffi_platform.Struct(
             'OSVERSIONINFOEX',
             [('dwOSVersionInfoSize', rffi.UINT),
@@ -93,7 +111,8 @@
                        PROCESS_VM_WRITE
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
-                       WC_NO_BEST_FIT_CHARS
+                       WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
+                       STD_ERROR_HANDLE
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -444,3 +463,13 @@
         return rffi.cast(lltype.Signed, _GetConsoleOutputCP())
 
     _wenviron_items, _wgetenv, _wputenv = make_env_impls(win32=True)
+
+
+    _GetStdHandle = winexternal(
+        'GetStdHandle', [DWORD], HANDLE)
+
+    def GetStdHandle(handle_id):
+        return _GetStdHandle(handle_id)
+    CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
+    GetConsoleScreenBufferInfo = winexternal(
+        "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)

From pypy.commits at gmail.com  Fri Aug 26 13:17:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 10:17:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: add trailing comma
Message-ID: <57c079af.56421c0a.e1297.f65f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86580:e239bf39ca98
Date: 2016-08-26 19:16 +0200
http://bitbucket.org/pypy/pypy/changeset/e239bf39ca98/

Log:	add trailing comma

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -19,7 +19,7 @@
         'statvfs_result': 'app_posix.statvfs_result',
         'uname_result': 'app_posix.uname_result',
         'urandom': 'app_posix.urandom',
-        'terminal_size': 'app_posix.terminal_size'
+        'terminal_size': 'app_posix.terminal_size',
     }
     if os.name == 'nt':
         del appleveldefs['urandom'] # at interp on win32
@@ -75,7 +75,7 @@
         'abort': 'interp_posix.abort',
         'urandom': 'interp_posix.urandom',
         'device_encoding': 'interp_posix.device_encoding',
-        'get_terminal_size': 'interp_posix.get_terminal_size'
+        'get_terminal_size': 'interp_posix.get_terminal_size',
 
         'scandir': 'interp_scandir.scandir',
     }

From pypy.commits at gmail.com  Fri Aug 26 13:28:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 10:28:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: hg merge py3.5
Message-ID: <57c07c33.448e1c0a.f4301.007a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86581:1ffd7af6c0b2
Date: 2016-08-26 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/1ffd7af6c0b2/

Log:	hg merge py3.5

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -19,7 +19,7 @@
         'statvfs_result': 'app_posix.statvfs_result',
         'uname_result': 'app_posix.uname_result',
         'urandom': 'app_posix.urandom',
-        'terminal_size': 'app_posix.terminal_size'
+        'terminal_size': 'app_posix.terminal_size',
     }
     if os.name == 'nt':
         del appleveldefs['urandom'] # at interp on win32
@@ -75,7 +75,7 @@
         'abort': 'interp_posix.abort',
         'urandom': 'interp_posix.urandom',
         'device_encoding': 'interp_posix.device_encoding',
-        'get_terminal_size': 'interp_posix.get_terminal_size'
+        'get_terminal_size': 'interp_posix.get_terminal_size',
 
         'scandir': 'interp_scandir.scandir',
         'get_inheritable': 'interp_posix.get_inheritable',

From pypy.commits at gmail.com  Fri Aug 26 13:28:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 10:28:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: _socket.dup(). and
 _socket.fromfd()... which no longer exists at
Message-ID: <57c07c34.c41f1c0a.119a3.02d4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86582:e6181605e018
Date: 2016-08-26 19:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e6181605e018/

Log:	_socket.dup(). and _socket.fromfd()... which no longer exists at
	interp-level, easy

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -143,24 +143,11 @@
 @unwrap_spec(fd=int)
 def dup(space, fd):
     try:
-        newfd = rsocket.dup(fd)
+        newfd = rsocket.dup(fd, inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
     return space.wrap(newfd)
 
- at unwrap_spec(fd=int, family=int, type=int, proto=int)
-def fromfd(space, fd, family, type, proto=0):
-    """fromfd(fd, family, type[, proto]) -> socket object
-
-    Create a socket object from the given file descriptor.
-    The remaining arguments are the same as for socket().
-    """
-    try:
-        sock = rsocket.fromfd(fd, family, type, proto)
-    except SocketError as e:
-        raise converted_error(space, e)
-    return space.wrap(W_Socket(space, sock))
-
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
                       type  =rsocket.SOCK_STREAM,
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -546,11 +546,15 @@
         s.ioctl(_socket.SIO_KEEPALIVE_VALS, (1, 100, 100))
 
     def test_dup(self):
-        import _socket as socket
+        import _socket as socket, posix
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.bind(('localhost', 0))
         fd = socket.dup(s.fileno())
         assert s.fileno() != fd
+        assert posix.get_inheritable(s.fileno()) is False
+        assert posix.get_inheritable(fd) is False
+        posix.close(fd)
+        s.close()
 
     def test_dup_error(self):
         import _socket
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -372,13 +372,17 @@
         raise OSError(get_saved_errno(), '%s failed' % name)
     return result
 
- at replace_os_function('dup')
-def dup(fd, inheritable=True):
+def _dup(fd, inheritable=True):
     validate_fd(fd)
     if inheritable:
         res = c_dup(fd)
     else:
         res = c_dup_noninheritable(fd)
+    return res
+
+ at replace_os_function('dup')
+def dup(fd, inheritable=True):
+    res = _dup(fd, inheritable)
     return handle_posix_error('dup', res)
 
 @replace_os_function('dup2')
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1169,7 +1169,7 @@
                 make_socket(fd1, family, type, proto, SocketClass))
 
 if _c.WIN32:
-    def dup(fd):
+    def dup(fd, inheritable=True):
         with lltype.scoped_alloc(_c.WSAPROTOCOL_INFO, zero=True) as info:
             if _c.WSADuplicateSocket(fd, rwin32.GetCurrentProcessId(), info):
                 raise last_error()
@@ -1180,15 +1180,16 @@
                 raise last_error()
             return result
 else:
-    def dup(fd):
-        fd = _c.dup(fd)
+    def dup(fd, inheritable=True):
+        fd = rposix._dup(fd, inheritable)
         if fd < 0:
             raise last_error()
         return fd
 
-def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
+def fromfd(fd, family, type, proto=0, SocketClass=RSocket, inheritable=True):
     # Dup the fd so it and the socket can be closed independently
-    return make_socket(dup(fd), family, type, proto, SocketClass)
+    fd = dup(fd, inheritable=inheritable)
+    return make_socket(fd, family, type, proto, SocketClass)
 
 def getdefaulttimeout():
     return defaults.timeout

From pypy.commits at gmail.com  Fri Aug 26 14:03:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 11:03:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: copy the logic from 3.5 to
 inherit some file descriptors
Message-ID: <57c0847b.271ac20a.51b3b.cf06@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86583:f00bf6596800
Date: 2016-08-26 20:03 +0200
http://bitbucket.org/pypy/pypy/changeset/f00bf6596800/

Log:	copy the logic from 3.5 to inherit some file descriptors

diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.c b/pypy/module/_posixsubprocess/_posixsubprocess.c
--- a/pypy/module/_posixsubprocess/_posixsubprocess.c
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.c
@@ -106,6 +106,30 @@
 }
 
 
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable);   /* rposix.py */
+
+static int
+make_inheritable(long *py_fds_to_keep, ssize_t num_fds_to_keep,
+                 int errpipe_write)
+{
+    long i;
+
+    for (i = 0; i < num_fds_to_keep; ++i) {
+        long fd = py_fds_to_keep[i];
+        if (fd == errpipe_write) {
+            /* errpipe_write is part of py_fds_to_keep. It must be closed at
+               exec(), but kept open in the child process until exec() is
+               called. */
+            continue;
+        }
+        if (rpy_set_inheritable((int)fd, 1) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+
 /* Close all file descriptors in the range start_fd inclusive to
  * end_fd exclusive except for those in py_fds_to_keep.  If the
  * range defined by [start_fd, end_fd) is large this will take a
@@ -329,6 +353,9 @@
     /* Buffer large enough to hold a hex integer.  We can't malloc. */
     char hex_errno[sizeof(saved_errno)*2+1];
 
+    if (make_inheritable(py_fds_to_keep, num_fds_to_keep, errpipe_write) < 0)
+        goto error;
+
     /* Close parent's pipe ends. */
     if (p2cwrite != -1) {
         POSIX_CALL(close(p2cwrite));
@@ -352,26 +379,25 @@
        dup2() removes the CLOEXEC flag but we must do it ourselves if dup2()
        would be a no-op (issue #10806). */
     if (p2cread == 0) {
-        int old = fcntl(p2cread, F_GETFD);
-        if (old != -1)
-            fcntl(p2cread, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (p2cread != -1) {
+        if (rpy_set_inheritable(p2cread, 1) < 0)
+            goto error;
+    }
+    else if (p2cread != -1)
         POSIX_CALL(dup2(p2cread, 0));  /* stdin */
+
+    if (c2pwrite == 1) {
+        if (rpy_set_inheritable(c2pwrite, 1) < 0)
+            goto error;
     }
-    if (c2pwrite == 1) {
-        int old = fcntl(c2pwrite, F_GETFD);
-        if (old != -1)
-            fcntl(c2pwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (c2pwrite != -1) {
+    else if (c2pwrite != -1)
         POSIX_CALL(dup2(c2pwrite, 1));  /* stdout */
+
+    if (errwrite == 2) {
+        if (rpy_set_inheritable(errwrite, 1) < 0)
+            goto error;
     }
-    if (errwrite == 2) {
-        int old = fcntl(errwrite, F_GETFD);
-        if (old != -1)
-            fcntl(errwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (errwrite != -1) {
+    else if (errwrite != -1)
         POSIX_CALL(dup2(errwrite, 2));  /* stderr */
-    }
 
     /* Close pipe fds.  Make sure we don't close the same fd more than */
     /* once, or standard fds. */
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.h b/pypy/module/_posixsubprocess/_posixsubprocess.h
--- a/pypy/module/_posixsubprocess/_posixsubprocess.h
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.h
@@ -1,3 +1,4 @@
+#include <unistd.h>    /* for ssize_t */
 #include "src/precommondefs.h"
 
 RPY_EXTERN void
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -5,6 +5,7 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib import rposix
 
 from pypy.interpreter.error import (
     OperationError, exception_from_saved_errno, oefmt, wrap_oserror)
@@ -36,6 +37,7 @@
     compile_extra.append("-DHAVE_SETSID")
 
 eci = eci.merge(
+    rposix.eci_inheritable,
     ExternalCompilationInfo(
         compile_extra=compile_extra))
 
diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py
--- a/pypy/module/_posixsubprocess/test/test_subprocess.py
+++ b/pypy/module/_posixsubprocess/test/test_subprocess.py
@@ -75,3 +75,18 @@
         n = 1
         raises(OverflowError, _posixsubprocess.fork_exec,
                1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17)
+
+    def test_pass_fds_make_inheritable(self):
+        import subprocess, posix
+
+        fd1, fd2 = posix.pipe()
+        assert posix.get_inheritable(fd1) is False
+        assert posix.get_inheritable(fd2) is False
+
+        subprocess.check_call(['/usr/bin/env', 'python', '-c',
+                               'import os;os.write(%d,b"K")' % fd2],
+                              close_fds=True, pass_fds=[fd2])
+        res = posix.read(fd1, 1)
+        assert res == b"K"
+        posix.close(fd1)
+        posix.close(fd2)

From pypy.commits at gmail.com  Fri Aug 26 14:09:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 11:09:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: some XXXs
Message-ID: <57c085cc.cb7f1c0a.fa27d.0c7c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86584:d490b2af811a
Date: 2016-08-26 20:08 +0200
http://bitbucket.org/pypy/pypy/changeset/d490b2af811a/

Log:	some XXXs

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -554,6 +554,9 @@
     def putwin(self, filep):
         # filestar = ffi.new("FILE *", filep)
         return _check_ERR(lib.putwin(self._win, filep), "putwin")
+        # XXX CPython 3.5 says: We have to simulate this by writing to
+        # a temporary FILE*, then reading back, then writing to the
+        # argument stream.
 
     def redrawln(self, beg, num):
         return _check_ERR(lib.wredrawln(self._win, beg, num), "redrawln")
@@ -704,6 +707,7 @@
 
 
 def getwin(filep):
+    # XXX CPython 3.5: there's logic to use a temp file instead
     return Window(_check_NULL(lib.getwin(filep)))
 
 

From pypy.commits at gmail.com  Fri Aug 26 17:34:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 Aug 2016 14:34:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: Restore a comment,
 and fix float.__hash__() and complex.__hash__() as well
Message-ID: <57c0b5e9.05d71c0a.74e96.539a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86585:6a2473172193
Date: 2016-08-26 23:33 +0200
http://bitbucket.org/pypy/pypy/changeset/6a2473172193/

Log:	Restore a comment, and fix float.__hash__() and complex.__hash__()
	as well

diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -363,9 +363,10 @@
 
     def descr_hash(self, space):
         hashreal = _hash_float(space, self.realval)
-        hashimg = _hash_float(space, self.imagval)
-        combined = intmask(hashreal + 1000003 * hashimg)
-        return space.newint(combined)
+        hashimg = _hash_float(space, self.imagval)   # 0 if self.imagval == 0
+        h = intmask(hashreal + 1000003 * hashimg)
+        h -= (h == -1)
+        return space.newint(h)
 
     def descr_coerce(self, space, w_other):
         w_other = self._to_complex(space, w_other)
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -390,7 +390,9 @@
         return space.wrap(float2string(self.floatval, 'g', DTSF_STR_PRECISION))
 
     def descr_hash(self, space):
-        return space.wrap(_hash_float(space, self.floatval))
+        h = _hash_float(space, self.floatval)
+        h -= (h == -1)
+        return space.wrap(h)
 
     def descr_format(self, space, w_spec):
         return newformat.run_formatter(space, w_spec, "format_float", self)
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -363,6 +363,9 @@
 
     def descr_hash(self, space):
         # For compatibility with CPython, we special-case -1
+        # Make sure this is consistent with the hash of floats and longs.
+        # The complete list of built-in types whose hash should be
+        # consistent is: int, long, bool, float, complex.
         h = self.intval
         h -= (h == -1)  # No explicit condition, to avoid JIT bridges
         return wrapint(space, h)
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -611,3 +611,7 @@
 
     def test_complex_two_arguments(self):
         raises(TypeError, complex, 5, None)
+
+    def test_hash_minus_one(self):
+        assert hash(-1.0 + 0j) == -2
+        assert (-1.0 + 0j).__hash__() == -2
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -455,6 +455,10 @@
         else:
             assert False, 'did not raise'
 
+    def test_hash_minus_one(self):
+        assert hash(-1.0) == -2
+        assert (-1.0).__hash__() == -2
+
 
 class AppTestFloatHex:
     spaceconfig = {

From pypy.commits at gmail.com  Sat Aug 27 00:49:05 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 26 Aug 2016 21:49:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix memoryview.__setitem__
Message-ID: <57c11bc1.88711c0a.70789.add4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86586:bc8331369e2d
Date: 2016-08-27 05:48 +0100
http://bitbucket.org/pypy/pypy/changeset/bc8331369e2d/

Log:	Fix memoryview.__setitem__

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -91,8 +91,6 @@
                 step = 1
             if stop > self.getlength():
                 raise oefmt(space.w_IndexError, 'index out of range')
-        if step not in (0, 1):
-            raise oefmt(space.w_NotImplementedError, "")
         if step == 0:  # index only
             # TODO: this probably isn't very fast
             buf = SubBuffer(self.buf, start, self.itemsize)
@@ -113,7 +111,7 @@
         if space.isinstance_w(w_index, space.w_tuple):
             raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.buf.getitemsize()
+        itemsize = self.itemsize
         if itemsize > 1:
             start *= itemsize
             size *= itemsize
@@ -122,12 +120,6 @@
                 step = 1
             if stop > self.getlength():
                 raise oefmt(space.w_IndexError, 'index out of range')
-        if step not in (0, 1):
-            raise oefmt(space.w_NotImplementedError, "")
-        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-        if value.getlength() != size:
-            raise oefmt(space.w_ValueError,
-                        "cannot modify size of memoryview object")
         if step == 0:  # index only
             # TODO: this probably isn't very fast
             fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)

From pypy.commits at gmail.com  Sat Aug 27 00:58:55 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 26 Aug 2016 21:58:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Restore support for the memoryview
 attributes that were already implemented
Message-ID: <57c11e0f.434bc20a.8dc9.2db8@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86587:8f0af3b2bb3e
Date: 2016-08-27 05:58 +0100
http://bitbucket.org/pypy/pypy/changeset/8f0af3b2bb3e/

Log:	Restore support for the memoryview attributes that were already
	implemented

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -145,11 +145,11 @@
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getformat())
+        return space.wrap(self.format)
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getitemsize())
+        return space.newint(self.itemsize)
 
     def w_get_ndim(self, space):
         self._check_released(space)
@@ -161,11 +161,11 @@
 
     def w_get_shape(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(x) for x in self.buf.getshape()])
+        return space.newtuple([space.newint(self.getlength())])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(x) for x in self.buf.getstrides()])
+        return space.newtuple([space.newint(self.itemsize)])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)

From pypy.commits at gmail.com  Sat Aug 27 02:56:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 26 Aug 2016 23:56:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: passing cast test with empty
 buffer
Message-ID: <57c139af.c41f1c0a.119a3.c598@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86588:b01af276a91a
Date: 2016-08-27 08:09 +0200
http://bitbucket.org/pypy/pypy/changeset/b01af276a91a/

Log:	passing cast test with empty buffer

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -48,7 +48,7 @@
     def buffer_w_ex(self, space, flags):
         self._check_released(space)
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf, self.format, self.itemsize
+        return self.buf, self.get_format(), self.itemsize
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
@@ -371,7 +371,7 @@
                 raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: cast must be 1D -> ND or ND -> 1D"))
 
-        mv = W_MemoryView(buf, fmt, itemsize)
+        mv = W_MemoryView(buf, self.format, self.itemsize)
         origfmt = mv.getformat()
         mv._cast_to_1D(space, origfmt, fmt, itemsize)
         if w_shape:
@@ -381,7 +381,7 @@
 
     def _init_flags(self):
         buf = self.buf
-        ndim = buf.ndim
+        ndim = buf.getndim()
         flags = 0
         if ndim == 0:
             flags |= MEMORYVIEW_SCALAR | MEMORYVIEW_C | MEMORYVIEW_FORTRAN
@@ -391,10 +391,12 @@
             if len(shape) > 0 and shape[0] == 1 and \
                len(strides) > 0 and strides[0] == buf.getitemsize():
                 flags |= MEMORYVIEW_C | MEMORYVIEW_SCALAR
-        if buf.is_contiguous('C'):
-            flags |= MEMORYVIEW_C
-        elif buf.is_contiguous('F'):
-            flags |= MEMORYVIEW_FORTRAN
+        # TODO for now?
+        flags |= MEMORYVIEW_C
+        # TODO if buf.is_contiguous('C'):
+        # TODO     flags |= MEMORYVIEW_C
+        # TODO elif buf.is_contiguous('F'):
+        # TODO     flags |= MEMORYVIEW_FORTRAN
 
         # XXX missing suboffsets
 
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -256,6 +256,9 @@
         return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
                           self.w_size, self.w_strides, self.w_shape)
 
+    def buffer_w_ex(self, space, flags):
+        return self.buffer_w(space, flags), space.str_w(self.w_fmt), space.int_w(self.w_size)
+
 W_MockArray.typedef = TypeDef("MockArray",
     __new__ = interp2app(W_MockArray.descr_new),
 )
@@ -317,11 +320,12 @@
         assert view.format == 'b'
         assert cview.format == 'i'
         #
-        assert cview.cast('i').cast('b').cast('i').tolist() == []
+        #assert cview.cast('i').cast('b').cast('i').tolist() == []
         #
+        assert cview.format == 'i'
         try:
-            cview = view.cast('i')
-            assert False, "cannot cast between two non byte formats!"
+            cview.cast('i')
+            assert False, "cast must fail"
         except TypeError:
             pass
 

From pypy.commits at gmail.com  Sat Aug 27 02:56:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 26 Aug 2016 23:56:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: making all memoryobject tests
 pass. resolve several issues in get/setitem
Message-ID: <57c139b1.274fc20a.2eca5.9418@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86589:72bfb3417856
Date: 2016-08-27 08:56 +0200
http://bitbucket.org/pypy/pypy/changeset/72bfb3417856/

Log:	making all memoryobject tests pass. resolve several issues in
	get/setitem

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -496,7 +496,7 @@
     return data
 
 HEXDIGITS = "0123456789abcdef"
-PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8)-1)
+PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8-1)-1)
 
 @specialize.arg(3) # raw access
 def _array_to_hexstring(space, buf, len=0, rawaccess=False):
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -48,7 +48,7 @@
     def buffer_w_ex(self, space, flags):
         self._check_released(space)
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf, self.get_format(), self.itemsize
+        return self.buf, self.getformat(), self.itemsize
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
@@ -157,16 +157,17 @@
         if space.isinstance_w(w_index, space.w_tuple):
             return self._getitem_tuple_indexed(space, w_index)
 
-        start, stop, step, size = space.decode_index4_or_tuple_index(w_index, \
-                                                              self.getlength())
-        itemsize = self.buf.getitemsize()
+        start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        itemsize = self.itemsize
         if itemsize > 1:
             start *= itemsize
             size *= itemsize
             stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
+            # XXX why? returns a memory view on int index if step == 0:
+            #    step = 1
+
+            # start & stop are now byte offset, thus use self.bug.getlength()
+            if stop > self.buf.getlength():
                 raise oefmt(space.w_IndexError, 'index out of range')
         if step not in (0, 1):
             raise oefmt(space.w_NotImplementedError, "")
@@ -190,21 +191,19 @@
         if space.isinstance_w(w_index, space.w_tuple):
             raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.buf.getitemsize()
+        itemsize = self.itemsize
         if itemsize > 1:
             start *= itemsize
             size *= itemsize
             stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
+            # XXX why? returns a memory view on int index if step == 0:
+            #    step = 1
+
+            # start & stop are now byte offset, thus use self.bug.getlength()
+            if stop > self.buf.getlength():
                 raise oefmt(space.w_IndexError, 'index out of range')
         if step not in (0, 1):
             raise oefmt(space.w_NotImplementedError, "")
-        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-        if value.getlength() != size:
-            raise oefmt(space.w_ValueError,
-                        "cannot modify size of memoryview object")
         if step == 0:  # index only
             # TODO: this probably isn't very fast
             fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)
@@ -226,7 +225,7 @@
 
     def descr_len(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getlength())
+        return space.wrap(self.getlength())
 
     def w_get_format(self, space):
         self._check_released(space)
@@ -234,7 +233,7 @@
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getitemsize())
+        return space.wrap(self.itemsize)
 
     def w_get_ndim(self, space):
         self._check_released(space)
@@ -426,7 +425,7 @@
         self.setformat(newfmt)
         self.itemsize = itemsize
         self.ndim = 1
-        self.shape = [buf.getlength() / buf.getitemsize()]
+        self.shape = [buf.getlength() // buf.getitemsize()]
         self.srides = [buf.getitemsize()]
         # XX suboffsets
 
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -266,7 +266,7 @@
 from pypy.objspace.std.transparent import register_proxyable
 from pypy.conftest import option
 
-class AppTestMemoryViewMicroNumPyPy(object):
+class AppTestMemoryViewMockBuffer(object):
     spaceconfig = dict(usemodules=[])
     def setup_class(cls):
         if option.runappdirect:

From pypy.commits at gmail.com  Sat Aug 27 04:02:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 01:02:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Backport memoryview.cast(), fix getitem
Message-ID: <57c1492f.08d11c0a.37497.d67d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86590:fbf9c57aa038
Date: 2016-08-27 09:46 +0200
http://bitbucket.org/pypy/pypy/changeset/fbf9c57aa038/

Log:	Backport memoryview.cast(), fix getitem

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -17,6 +17,7 @@
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
+    _immutable_fields_ = ['format', 'itemsize']
 
     def __init__(self, buf, format='B', itemsize=1):
         assert isinstance(buf, Buffer)
@@ -62,8 +63,7 @@
 
     def as_str(self):
         buf = self.buf
-        n_bytes = buf.getlength()
-        return buf.getslice(0, n_bytes, 1, n_bytes)
+        return buf.as_str()
 
     def getlength(self):
         return self.buf.getlength() // self.itemsize
@@ -82,27 +82,24 @@
     def descr_getitem(self, space, w_index):
         self._check_released(space)
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.buf.getitemsize()
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
+        # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
+        itemsize = self.itemsize
         if step == 0:  # index only
-            # TODO: this probably isn't very fast
-            buf = SubBuffer(self.buf, start, self.itemsize)
-            fmtiter = UnpackFormatIterator(space, buf)
-            fmtiter.interpret(self.format)
-            return fmtiter.result_w[0]
+            if itemsize == 1:
+                ch = self.buf.getitem(start)
+                return space.newint(ord(ch))
+            else:
+                # TODO: this probably isn't very fast
+                buf = SubBuffer(self.buf, start * itemsize, itemsize)
+                fmtiter = UnpackFormatIterator(space, buf)
+                fmtiter.interpret(self.format)
+                return fmtiter.result_w[0]
         elif step == 1:
-            buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf, self.format, self.itemsize)
+            buf = SubBuffer(self.buf, start * itemsize, size * itemsize)
+            return W_MemoryView(buf, self.format, itemsize)
         else:
-            buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf)
+            raise oefmt(space.w_NotImplementedError,
+                        "XXX extended slicing")
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
@@ -141,7 +138,7 @@
 
     def descr_len(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getlength())
+        return space.wrap(self.getlength())
 
     def w_get_format(self, space):
         self._check_released(space)
@@ -214,6 +211,44 @@
             raise OperationError(space.w_ValueError, space.wrap(msg))
         return space.wrap(rffi.cast(lltype.Signed, ptr))
 
+    def get_native_fmtchar(self, fmt):
+        from rpython.rtyper.lltypesystem import rffi
+        size = -1
+        if fmt[0] == '@':
+            f = fmt[1]
+        else:
+            f = fmt[0]
+        if f == 'c' or f == 'b' or f == 'B':
+            size = rffi.sizeof(rffi.CHAR)
+        elif f == 'h' or f == 'H':
+            size = rffi.sizeof(rffi.SHORT)
+        elif f == 'i' or f == 'I':
+            size = rffi.sizeof(rffi.INT)
+        elif f == 'l' or f == 'L':
+            size = rffi.sizeof(rffi.LONG)
+        elif f == 'q' or f == 'Q':
+            size = rffi.sizeof(rffi.LONGLONG)
+        elif f == 'n' or f == 'N':
+            size = rffi.sizeof(rffi.SIZE_T)
+        elif f == 'f':
+            size = rffi.sizeof(rffi.FLOAT)
+        elif f == 'd':
+            size = rffi.sizeof(rffi.DOUBLE)
+        elif f == '?':
+            size = rffi.sizeof(rffi.CHAR)
+        elif f == 'P':
+            size = rffi.sizeof(rffi.VOIDP)
+        return size
+
+    def descr_cast(self, space, w_format, w_shape=None):
+        self._check_released(space)
+        if not space.is_none(w_shape):
+            raise oefmt(space.w_NotImplementedError,
+                        "XXX cast() with a shape")
+        fmt = space.str_w(w_format)
+        newitemsize = self.get_native_fmtchar(fmt)
+        return W_MemoryView(self.buf, fmt, newitemsize)
+
 W_MemoryView.typedef = TypeDef(
     "memoryview",
     __doc__ = """\
@@ -230,6 +265,7 @@
     __enter__   = interp2app(W_MemoryView.descr_enter),
     __exit__    = interp2app(W_MemoryView.descr_exit),
     __weakref__ = make_weakref_descr(W_MemoryView),
+    cast        = interp2app(W_MemoryView.descr_cast),
     tobytes     = interp2app(W_MemoryView.descr_tobytes),
     tolist      = interp2app(W_MemoryView.descr_tolist),
     release     = interp2app(W_MemoryView.descr_release),
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -16,8 +16,6 @@
         w = v[1:234]
         assert isinstance(w, memoryview)
         assert len(w) == 2
-        exc = raises(NotImplementedError, "v[0:2:2]")
-        assert str(exc.value) == ""
 
     def test_rw(self):
         data = bytearray(b'abcefg')
@@ -31,8 +29,15 @@
         assert data == bytearray(eval("b'23f3fg'"))
         exc = raises(ValueError, "v[2:3] = b'spam'")
         assert str(exc.value) == "cannot modify size of memoryview object"
-        exc = raises(NotImplementedError, "v[0:2:2] = b'spam'")
-        assert str(exc.value) == ""
+
+    def test_extended_slice(self):
+        data = bytearray(b'abcefg')
+        v = memoryview(data)
+        w = v[0:2:2]      # failing for now: NotImplementedError
+        assert len(w) == 1
+        assert list(w) == [97]
+        v[::2] = b'ABC'
+        assert data == bytearray(b'AbBeCg')
 
     def test_memoryview_attrs(self):
         v = memoryview(b"a"*100)
@@ -161,3 +166,33 @@
         raises(ValueError, memoryview(b"foobar")._pypy_raw_address)
         a = memoryview(bytearray(b"foobar"))._pypy_raw_address()
         assert a != 0
+
+    def test_memoryview_cast(self):
+        m1 = memoryview(b'abcdefgh')
+        m2 = m1.cast('I')
+        m3 = m1.cast('h')
+        assert list(m1) == [97, 98, 99, 100, 101, 102, 103, 104]
+        assert list(m2) == [1684234849, 1751606885]
+        assert list(m3) == [25185, 25699, 26213, 26727]
+        assert m1[1] == 98
+        assert m2[1] == 1751606885
+        assert m3[1] == 25699
+        assert list(m3[1:3]) == [25699, 26213]
+        assert m3[1:3].tobytes() == b'cdef'
+        assert len(m2) == 2
+        assert len(m3) == 4
+        assert (m2[-2], m2[-1]) == (1684234849, 1751606885)
+        raises(IndexError, "m2[2]")
+        raises(IndexError, "m2[-3]")
+        assert list(m3[-99:3]) == [25185, 25699, 26213]
+        assert list(m3[1:99]) == [25699, 26213, 26727]
+        raises(IndexError, "m1[8]")
+        raises(IndexError, "m1[-9]")
+        assert m1[-8] == 97
+
+    def test_memoryview_cast_extended_slicing(self):
+        m1 = memoryview(b'abcdefgh')
+        m3 = m1.cast('h')
+        assert m3[1::2].tobytes() == b'cdgh'
+        assert m3[::2].tobytes() == b'abef'
+        assert m3[:2:2].tobytes() == b'ab'
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -10,6 +10,7 @@
     _immutable_ = True
 
     def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
         raise NotImplementedError
 
     def __len__(self):

From pypy.commits at gmail.com  Sat Aug 27 04:02:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 01:02:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: memoryview.setitem()
Message-ID: <57c14931.0cce1c0a.b81b5.d562@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86591:8c171091c731
Date: 2016-08-27 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/8c171091c731/

Log:	memoryview.setitem()

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -10,6 +10,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
+from pypy.objspace.std.bytesobject import getbytevalue
 from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
 
 
@@ -106,35 +107,32 @@
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         if space.isinstance_w(w_index, space.w_tuple):
-            raise oefmt(space.w_NotImplementedError, "")
+            raise oefmt(space.w_NotImplementedError, "XXX tuple setitem")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         itemsize = self.itemsize
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
         if step == 0:  # index only
-            # TODO: this probably isn't very fast
-            fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)
-            try:
-                fmtiter.interpret(self.format)
-            except StructError as e:
-                raise oefmt(space.w_TypeError,
-                            "memoryview: invalid type for format '%s'",
-                            self.format)
-            self.buf.setslice(start, fmtiter.result.build())
+            if itemsize == 1:
+                ch = getbytevalue(space, w_obj)
+                self.buf.setitem(start, ch)
+            else:
+                # TODO: this probably isn't very fast
+                fmtiter = PackFormatIterator(space, [w_obj], itemsize)
+                try:
+                    fmtiter.interpret(self.format)
+                except StructError as e:
+                    raise oefmt(space.w_TypeError,
+                                "memoryview: invalid type for format '%s'",
+                                self.format)
+                self.buf.setslice(start * itemsize, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
             if value.getlength() != size * self.itemsize:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
-            self.buf.setslice(start, value.as_str())
+            self.buf.setslice(start * itemsize, value.as_str())
         else:
-            raise oefmt(space.w_NotImplementedError, "")
+            raise oefmt(space.w_NotImplementedError,
+                        "XXX extended slicing")
 
     def descr_len(self, space):
         self._check_released(space)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -196,3 +196,21 @@
         assert m3[1::2].tobytes() == b'cdgh'
         assert m3[::2].tobytes() == b'abef'
         assert m3[:2:2].tobytes() == b'ab'
+
+    def test_memoryview_cast_setitem(self):
+        data = bytearray(b'abcdefgh')
+        m1 = memoryview(data)
+        m2 = m1.cast('I')
+        m3 = m1.cast('h')
+        m1[2] = ord(b'C')
+        assert m2[0] == 1682137697
+        m3[1] = -9999
+        assert data == bytearray(bytes([97, 98, 241, 216, 101, 102, 103, 104]))
+        m3[1:3] = memoryview(b"pqrs").cast('h')
+        assert data == bytearray(b'abpqrsgh')
+
+    def test_memoryview_cast_setitem_extended_slicing(self):
+        data = bytearray(b'abcdefghij')
+        m3 = memoryview(data).cast('h')
+        m3[1:5:2] = memoryview(b"xyXY").cast('h')
+        assert data == bytearray(b'abxyefXYij')

From pypy.commits at gmail.com  Sat Aug 27 04:09:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 01:09:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c14ab6.12331c0a.ee8e2.db65@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86592:12c05bf05cd7
Date: 2016-08-27 10:05 +0200
http://bitbucket.org/pypy/pypy/changeset/12c05bf05cd7/

Log:	hg merge py3k

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -10,6 +10,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
+from pypy.objspace.std.bytesobject import getbytevalue
 from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
 
 
@@ -17,6 +18,7 @@
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
+    _immutable_fields_ = ['format', 'itemsize']
 
     def __init__(self, buf, format='B', itemsize=1):
         assert isinstance(buf, Buffer)
@@ -62,8 +64,7 @@
 
     def as_str(self):
         buf = self.buf
-        n_bytes = buf.getlength()
-        return buf.getslice(0, n_bytes, 1, n_bytes)
+        return buf.as_str()
 
     def getlength(self):
         return self.buf.getlength() // self.itemsize
@@ -82,82 +83,68 @@
     def descr_getitem(self, space, w_index):
         self._check_released(space)
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.buf.getitemsize()
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
-        if step not in (0, 1):
-            raise oefmt(space.w_NotImplementedError, "")
+        # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
+        itemsize = self.itemsize
         if step == 0:  # index only
-            # TODO: this probably isn't very fast
-            buf = SubBuffer(self.buf, start, self.itemsize)
-            fmtiter = UnpackFormatIterator(space, buf)
-            fmtiter.interpret(self.format)
-            return fmtiter.result_w[0]
+            if itemsize == 1:
+                ch = self.buf.getitem(start)
+                return space.newint(ord(ch))
+            else:
+                # TODO: this probably isn't very fast
+                buf = SubBuffer(self.buf, start * itemsize, itemsize)
+                fmtiter = UnpackFormatIterator(space, buf)
+                fmtiter.interpret(self.format)
+                return fmtiter.result_w[0]
         elif step == 1:
-            buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf, self.format, self.itemsize)
+            buf = SubBuffer(self.buf, start * itemsize, size * itemsize)
+            return W_MemoryView(buf, self.format, itemsize)
         else:
-            buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf)
+            raise oefmt(space.w_NotImplementedError,
+                        "XXX extended slicing")
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         if space.isinstance_w(w_index, space.w_tuple):
-            raise oefmt(space.w_NotImplementedError, "")
+            raise oefmt(space.w_NotImplementedError, "XXX tuple setitem")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.buf.getitemsize()
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
-        if step not in (0, 1):
-            raise oefmt(space.w_NotImplementedError, "")
-        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-        if value.getlength() != size:
-            raise oefmt(space.w_ValueError,
-                        "cannot modify size of memoryview object")
+        itemsize = self.itemsize
         if step == 0:  # index only
-            # TODO: this probably isn't very fast
-            fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)
-            try:
-                fmtiter.interpret(self.format)
-            except StructError as e:
-                raise oefmt(space.w_TypeError,
-                            "memoryview: invalid type for format '%s'",
-                            self.format)
-            self.buf.setslice(start, fmtiter.result.build())
+            if itemsize == 1:
+                ch = getbytevalue(space, w_obj)
+                self.buf.setitem(start, ch)
+            else:
+                # TODO: this probably isn't very fast
+                fmtiter = PackFormatIterator(space, [w_obj], itemsize)
+                try:
+                    fmtiter.interpret(self.format)
+                except StructError as e:
+                    raise oefmt(space.w_TypeError,
+                                "memoryview: invalid type for format '%s'",
+                                self.format)
+                self.buf.setslice(start * itemsize, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
             if value.getlength() != size * self.itemsize:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
-            self.buf.setslice(start, value.as_str())
+            self.buf.setslice(start * itemsize, value.as_str())
         else:
-            raise oefmt(space.w_NotImplementedError, "")
+            raise oefmt(space.w_NotImplementedError,
+                        "XXX extended slicing")
 
     def descr_len(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getlength())
+        return space.wrap(self.getlength())
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getformat())
+        return space.wrap(self.format)
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getitemsize())
+        return space.newint(self.itemsize)
 
     def w_get_ndim(self, space):
         self._check_released(space)
@@ -169,11 +156,11 @@
 
     def w_get_shape(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(x) for x in self.buf.getshape()])
+        return space.newtuple([space.newint(self.getlength())])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(x) for x in self.buf.getstrides()])
+        return space.newtuple([space.newint(self.itemsize)])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)
@@ -221,7 +208,7 @@
                    "is internally %r" % (self.buf,))
             raise OperationError(space.w_ValueError, space.wrap(msg))
         return space.wrap(rffi.cast(lltype.Signed, ptr))
-    
+
     def get_native_fmtchar(self, fmt):
         from rpython.rtyper.lltypesystem import rffi
         size = -1
@@ -252,8 +239,10 @@
         return size
 
     def descr_cast(self, space, w_format, w_shape=None):
-        # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
         self._check_released(space)
+        if not space.is_none(w_shape):
+            raise oefmt(space.w_NotImplementedError,
+                        "XXX cast() with a shape")
         fmt = space.str_w(w_format)
         newitemsize = self.get_native_fmtchar(fmt)
         return W_MemoryView(self.buf, fmt, newitemsize)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -16,8 +16,6 @@
         w = v[1:234]
         assert isinstance(w, memoryview)
         assert len(w) == 2
-        exc = raises(NotImplementedError, "v[0:2:2]")
-        assert str(exc.value) == ""
         exc = raises(TypeError, "memoryview('foobar')")
 
     def test_rw(self):
@@ -32,8 +30,15 @@
         assert data == bytearray(eval("b'23f3fg'"))
         exc = raises(ValueError, "v[2:3] = b'spam'")
         assert str(exc.value) == "cannot modify size of memoryview object"
-        exc = raises(NotImplementedError, "v[0:2:2] = b'spam'")
-        assert str(exc.value) == ""
+
+    def test_extended_slice(self):
+        data = bytearray(b'abcefg')
+        v = memoryview(data)
+        w = v[0:2:2]      # failing for now: NotImplementedError
+        assert len(w) == 1
+        assert list(w) == [97]
+        v[::2] = b'ABC'
+        assert data == bytearray(b'AbBeCg')
 
     def test_memoryview_attrs(self):
         v = memoryview(b"a"*100)
@@ -165,3 +170,51 @@
 
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
+
+    def test_memoryview_cast(self):
+        m1 = memoryview(b'abcdefgh')
+        m2 = m1.cast('I')
+        m3 = m1.cast('h')
+        assert list(m1) == [97, 98, 99, 100, 101, 102, 103, 104]
+        assert list(m2) == [1684234849, 1751606885]
+        assert list(m3) == [25185, 25699, 26213, 26727]
+        assert m1[1] == 98
+        assert m2[1] == 1751606885
+        assert m3[1] == 25699
+        assert list(m3[1:3]) == [25699, 26213]
+        assert m3[1:3].tobytes() == b'cdef'
+        assert len(m2) == 2
+        assert len(m3) == 4
+        assert (m2[-2], m2[-1]) == (1684234849, 1751606885)
+        raises(IndexError, "m2[2]")
+        raises(IndexError, "m2[-3]")
+        assert list(m3[-99:3]) == [25185, 25699, 26213]
+        assert list(m3[1:99]) == [25699, 26213, 26727]
+        raises(IndexError, "m1[8]")
+        raises(IndexError, "m1[-9]")
+        assert m1[-8] == 97
+
+    def test_memoryview_cast_extended_slicing(self):
+        m1 = memoryview(b'abcdefgh')
+        m3 = m1.cast('h')
+        assert m3[1::2].tobytes() == b'cdgh'
+        assert m3[::2].tobytes() == b'abef'
+        assert m3[:2:2].tobytes() == b'ab'
+
+    def test_memoryview_cast_setitem(self):
+        data = bytearray(b'abcdefgh')
+        m1 = memoryview(data)
+        m2 = m1.cast('I')
+        m3 = m1.cast('h')
+        m1[2] = ord(b'C')
+        assert m2[0] == 1682137697
+        m3[1] = -9999
+        assert data == bytearray(bytes([97, 98, 241, 216, 101, 102, 103, 104]))
+        m3[1:3] = memoryview(b"pqrs").cast('h')
+        assert data == bytearray(b'abpqrsgh')
+
+    def test_memoryview_cast_setitem_extended_slicing(self):
+        data = bytearray(b'abcdefghij')
+        m3 = memoryview(data).cast('h')
+        m3[1:5:2] = memoryview(b"xyXY").cast('h')
+        assert data == bytearray(b'abxyefXYij')
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -10,6 +10,7 @@
     _immutable_ = True
 
     def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
         raise NotImplementedError
 
     def __len__(self):

From pypy.commits at gmail.com  Sat Aug 27 04:09:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 01:09:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: small fix to hex()
Message-ID: <57c14ab8.8a13c20a.23ed6.9e81@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86593:9fc4d6946a5b
Date: 2016-08-27 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/9fc4d6946a5b/

Log:	small fix to hex()

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1,5 +1,6 @@
 """The builtin bytearray implementation"""
 
+import sys
 from rpython.rlib.objectmodel import (
     import_from_mixin, newlist_hint, resizelist_hint, specialize)
 from rpython.rlib.buffer import Buffer
@@ -496,7 +497,6 @@
     return data
 
 HEXDIGITS = "0123456789abcdef"
-PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8)-1)
 
 @specialize.arg(3) # raw access
 def _array_to_hexstring(space, buf, len=0, rawaccess=False):
@@ -504,11 +504,11 @@
         length = len
     else:
         length = buf.getlength()
-    hexstring = StringBuilder(length*2)
 
-    if length > PY_SIZE_T_MAX/2:
+    if length > sys.maxint / 2:
         raise OperationError(space.w_MemoryError, space.w_None)
 
+    hexstring = StringBuilder(length*2)
     for i in range(length):
         if rawaccess:
             byte = ord(buf[i])

From pypy.commits at gmail.com  Sat Aug 27 04:17:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 01:17:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: hg merge py3.5
Message-ID: <57c14c91.274fc20a.2eca5.ac24@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86594:2a96f5773e56
Date: 2016-08-27 10:16 +0200
http://bitbucket.org/pypy/pypy/changeset/2a96f5773e56/

Log:	hg merge py3.5

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1,5 +1,6 @@
 """The builtin bytearray implementation"""
 
+import sys
 from rpython.rlib.objectmodel import (
     import_from_mixin, newlist_hint, resizelist_hint, specialize)
 from rpython.rlib.buffer import Buffer
@@ -496,7 +497,6 @@
     return data
 
 HEXDIGITS = "0123456789abcdef"
-PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8)-1)
 
 @specialize.arg(3) # raw access
 def _array_to_hexstring(space, buf, len=0, rawaccess=False):
@@ -504,11 +504,11 @@
         length = len
     else:
         length = buf.getlength()
-    hexstring = StringBuilder(length*2)
 
-    if length > PY_SIZE_T_MAX/2:
+    if length > sys.maxint / 2:
         raise OperationError(space.w_MemoryError, space.w_None)
 
+    hexstring = StringBuilder(length*2)
     for i in range(length):
         if rawaccess:
             byte = ord(buf[i])
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -10,6 +10,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
+from pypy.objspace.std.bytesobject import getbytevalue
 from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
 
 
@@ -17,6 +18,7 @@
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
+    _immutable_fields_ = ['format', 'itemsize']
 
     def __init__(self, buf, format='B', itemsize=1):
         assert isinstance(buf, Buffer)
@@ -62,8 +64,7 @@
 
     def as_str(self):
         buf = self.buf
-        n_bytes = buf.getlength()
-        return buf.getslice(0, n_bytes, 1, n_bytes)
+        return buf.as_str()
 
     def getlength(self):
         return self.buf.getlength() // self.itemsize
@@ -82,82 +83,68 @@
     def descr_getitem(self, space, w_index):
         self._check_released(space)
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.buf.getitemsize()
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
-        if step not in (0, 1):
-            raise oefmt(space.w_NotImplementedError, "")
+        # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
+        itemsize = self.itemsize
         if step == 0:  # index only
-            # TODO: this probably isn't very fast
-            buf = SubBuffer(self.buf, start, self.itemsize)
-            fmtiter = UnpackFormatIterator(space, buf)
-            fmtiter.interpret(self.format)
-            return fmtiter.result_w[0]
+            if itemsize == 1:
+                ch = self.buf.getitem(start)
+                return space.newint(ord(ch))
+            else:
+                # TODO: this probably isn't very fast
+                buf = SubBuffer(self.buf, start * itemsize, itemsize)
+                fmtiter = UnpackFormatIterator(space, buf)
+                fmtiter.interpret(self.format)
+                return fmtiter.result_w[0]
         elif step == 1:
-            buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf, self.format, self.itemsize)
+            buf = SubBuffer(self.buf, start * itemsize, size * itemsize)
+            return W_MemoryView(buf, self.format, itemsize)
         else:
-            buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf)
+            raise oefmt(space.w_NotImplementedError,
+                        "XXX extended slicing")
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         if space.isinstance_w(w_index, space.w_tuple):
-            raise oefmt(space.w_NotImplementedError, "")
+            raise oefmt(space.w_NotImplementedError, "XXX tuple setitem")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.buf.getitemsize()
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
-        if step not in (0, 1):
-            raise oefmt(space.w_NotImplementedError, "")
-        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-        if value.getlength() != size:
-            raise oefmt(space.w_ValueError,
-                        "cannot modify size of memoryview object")
+        itemsize = self.itemsize
         if step == 0:  # index only
-            # TODO: this probably isn't very fast
-            fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)
-            try:
-                fmtiter.interpret(self.format)
-            except StructError as e:
-                raise oefmt(space.w_TypeError,
-                            "memoryview: invalid type for format '%s'",
-                            self.format)
-            self.buf.setslice(start, fmtiter.result.build())
+            if itemsize == 1:
+                ch = getbytevalue(space, w_obj)
+                self.buf.setitem(start, ch)
+            else:
+                # TODO: this probably isn't very fast
+                fmtiter = PackFormatIterator(space, [w_obj], itemsize)
+                try:
+                    fmtiter.interpret(self.format)
+                except StructError as e:
+                    raise oefmt(space.w_TypeError,
+                                "memoryview: invalid type for format '%s'",
+                                self.format)
+                self.buf.setslice(start * itemsize, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
             if value.getlength() != size * self.itemsize:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
-            self.buf.setslice(start, value.as_str())
+            self.buf.setslice(start * itemsize, value.as_str())
         else:
-            raise oefmt(space.w_NotImplementedError, "")
+            raise oefmt(space.w_NotImplementedError,
+                        "XXX extended slicing")
 
     def descr_len(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getlength())
+        return space.wrap(self.getlength())
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getformat())
+        return space.wrap(self.format)
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getitemsize())
+        return space.newint(self.itemsize)
 
     def w_get_ndim(self, space):
         self._check_released(space)
@@ -169,11 +156,11 @@
 
     def w_get_shape(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(x) for x in self.buf.getshape()])
+        return space.newtuple([space.newint(self.getlength())])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(x) for x in self.buf.getstrides()])
+        return space.newtuple([space.newint(self.itemsize)])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)
@@ -221,7 +208,7 @@
                    "is internally %r" % (self.buf,))
             raise OperationError(space.w_ValueError, space.wrap(msg))
         return space.wrap(rffi.cast(lltype.Signed, ptr))
-    
+
     def get_native_fmtchar(self, fmt):
         from rpython.rtyper.lltypesystem import rffi
         size = -1
@@ -252,8 +239,10 @@
         return size
 
     def descr_cast(self, space, w_format, w_shape=None):
-        # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
         self._check_released(space)
+        if not space.is_none(w_shape):
+            raise oefmt(space.w_NotImplementedError,
+                        "XXX cast() with a shape")
         fmt = space.str_w(w_format)
         newitemsize = self.get_native_fmtchar(fmt)
         return W_MemoryView(self.buf, fmt, newitemsize)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -16,8 +16,6 @@
         w = v[1:234]
         assert isinstance(w, memoryview)
         assert len(w) == 2
-        exc = raises(NotImplementedError, "v[0:2:2]")
-        assert str(exc.value) == ""
         exc = raises(TypeError, "memoryview('foobar')")
 
     def test_rw(self):
@@ -32,8 +30,15 @@
         assert data == bytearray(eval("b'23f3fg'"))
         exc = raises(ValueError, "v[2:3] = b'spam'")
         assert str(exc.value) == "cannot modify size of memoryview object"
-        exc = raises(NotImplementedError, "v[0:2:2] = b'spam'")
-        assert str(exc.value) == ""
+
+    def test_extended_slice(self):
+        data = bytearray(b'abcefg')
+        v = memoryview(data)
+        w = v[0:2:2]      # failing for now: NotImplementedError
+        assert len(w) == 1
+        assert list(w) == [97]
+        v[::2] = b'ABC'
+        assert data == bytearray(b'AbBeCg')
 
     def test_memoryview_attrs(self):
         v = memoryview(b"a"*100)
@@ -165,3 +170,51 @@
 
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
+
+    def test_memoryview_cast(self):
+        m1 = memoryview(b'abcdefgh')
+        m2 = m1.cast('I')
+        m3 = m1.cast('h')
+        assert list(m1) == [97, 98, 99, 100, 101, 102, 103, 104]
+        assert list(m2) == [1684234849, 1751606885]
+        assert list(m3) == [25185, 25699, 26213, 26727]
+        assert m1[1] == 98
+        assert m2[1] == 1751606885
+        assert m3[1] == 25699
+        assert list(m3[1:3]) == [25699, 26213]
+        assert m3[1:3].tobytes() == b'cdef'
+        assert len(m2) == 2
+        assert len(m3) == 4
+        assert (m2[-2], m2[-1]) == (1684234849, 1751606885)
+        raises(IndexError, "m2[2]")
+        raises(IndexError, "m2[-3]")
+        assert list(m3[-99:3]) == [25185, 25699, 26213]
+        assert list(m3[1:99]) == [25699, 26213, 26727]
+        raises(IndexError, "m1[8]")
+        raises(IndexError, "m1[-9]")
+        assert m1[-8] == 97
+
+    def test_memoryview_cast_extended_slicing(self):
+        m1 = memoryview(b'abcdefgh')
+        m3 = m1.cast('h')
+        assert m3[1::2].tobytes() == b'cdgh'
+        assert m3[::2].tobytes() == b'abef'
+        assert m3[:2:2].tobytes() == b'ab'
+
+    def test_memoryview_cast_setitem(self):
+        data = bytearray(b'abcdefgh')
+        m1 = memoryview(data)
+        m2 = m1.cast('I')
+        m3 = m1.cast('h')
+        m1[2] = ord(b'C')
+        assert m2[0] == 1682137697
+        m3[1] = -9999
+        assert data == bytearray(bytes([97, 98, 241, 216, 101, 102, 103, 104]))
+        m3[1:3] = memoryview(b"pqrs").cast('h')
+        assert data == bytearray(b'abpqrsgh')
+
+    def test_memoryview_cast_setitem_extended_slicing(self):
+        data = bytearray(b'abcdefghij')
+        m3 = memoryview(data).cast('h')
+        m3[1:5:2] = memoryview(b"xyXY").cast('h')
+        assert data == bytearray(b'abxyefXYij')
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -10,6 +10,7 @@
     _immutable_ = True
 
     def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
         raise NotImplementedError
 
     def __len__(self):

From pypy.commits at gmail.com  Sat Aug 27 05:14:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 02:14:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: Translation fixes
Message-ID: <57c159fb.d4e01c0a.e9e5b.f3a6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86595:93d0ce98997d
Date: 2016-08-27 11:14 +0200
http://bitbucket.org/pypy/pypy/changeset/93d0ce98997d/

Log:	Translation fixes

diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -544,7 +544,8 @@
 socketaccept = external('accept', [socketfd_type, sockaddr_ptr,
                                    socklen_t_ptr], socketfd_type,
                         save_err=SAVE_ERR)
-if cConfig.HAVE_ACCEPT4:
+HAVE_ACCEPT4 = cConfig.HAVE_ACCEPT4
+if HAVE_ACCEPT4:
     socketaccept4 = external('accept4', [socketfd_type, sockaddr_ptr,
                                          socklen_t_ptr, rffi.INT],
                                         socketfd_type,
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # XXX this does not support yet the least common AF_xxx address families
 # supported by CPython.  See http://bugs.pypy.org/issue1942
 
+from errno import EINVAL
 from rpython.rlib import _rsocket_rffi as _c, jit, rgc
 from rpython.rlib.objectmodel import instantiate, keepalive_until_here
 from rpython.rlib.rarithmetic import intmask, r_uint
@@ -531,7 +532,7 @@
                 # then we fall back to the SOCK_CLOEXEC-less case.
                 fd = _c.socket(family, type | SOCK_CLOEXEC, proto)
                 if fd < 0:
-                    if _c.geterrno() == errno.EINVAL:
+                    if _c.geterrno() == EINVAL:
                         # Linux older than 2.6.27 does not support
                         # SOCK_CLOEXEC.  An EINVAL might be caused by
                         # random other things, though.  Don't cache.
@@ -655,8 +656,8 @@
         try:
             remove_inheritable = not inheritable
             if (not inheritable and SOCK_CLOEXEC is not None
-                    and hasattr(_c, 'socketaccept4') and
-                    _accept4_syscall.attempt_syscall()):
+                    and _c.HAVE_ACCEPT4
+                    and _accept4_syscall.attempt_syscall()):
                 newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
                                          SOCK_CLOEXEC)
                 if _accept4_syscall.fallback(newfd):
@@ -1144,7 +1145,7 @@
                 res = _c.socketpair(family, type | SOCK_CLOEXEC,
                                     proto, result)
                 if res < 0:
-                    if _c.geterrno() == errno.EINVAL:
+                    if _c.geterrno() == EINVAL:
                         # Linux older than 2.6.27 does not support
                         # SOCK_CLOEXEC.  An EINVAL might be caused by
                         # random other things, though.  Don't cache.

From pypy.commits at gmail.com  Sat Aug 27 08:33:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 05:33:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: List the functions here
Message-ID: <57c18892.a719c20a.7a270.fe68@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86596:6623c68e0c68
Date: 2016-08-27 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/6623c68e0c68/

Log:	List the functions here

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -2242,7 +2242,10 @@
 #endif
 }
     """ % {'HAVE_DUP3': HAVE_DUP3}],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
 
 c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,

From pypy.commits at gmail.com  Sat Aug 27 08:42:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 05:42:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Write down the possible solution,
 to do later
Message-ID: <57c18ab8.109a1c0a.c338b.3824@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86597:a6da2166c78c
Date: 2016-08-27 14:42 +0200
http://bitbucket.org/pypy/pypy/changeset/a6da2166c78c/

Log:	Write down the possible solution, to do later

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -99,6 +99,8 @@
             buf = SubBuffer(self.buf, start * itemsize, size * itemsize)
             return W_MemoryView(buf, self.format, itemsize)
         else:
+            # XXX needs to return a W_MemoryView with a NonContiguousSubBuffer
+            # maybe?  Need to check the cpyext requirements for that
             raise oefmt(space.w_NotImplementedError,
                         "XXX extended slicing")
 

From pypy.commits at gmail.com  Sat Aug 27 08:55:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 05:55:39 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Working on the marshal format
Message-ID: <57c18dcb.06b0c20a.5eba8.6149@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5684:3943ce39c27a
Date: 2016-08-27 14:55 +0200
http://bitbucket.org/pypy/extradoc/changeset/3943ce39c27a/

Log:	Working on the marshal format

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -13,6 +13,8 @@
 
 * arigo: Newly created file descriptors are non-inheritable (PEP 446)
 
+* arigo: New marshal format
+
 
 Finished
 --------
@@ -63,11 +65,7 @@
 
 * The new os.scandir() function (POSIX-DONE, missing Win32)
 
-* Newly created file descriptors are non-inheritable (PEP 446)
-
-  - added rposix.{set,get}_inheritable(), used it as a quick hack
-    inside interp_posix.pipe(), needed otherwise subprocess.Popen()
-    deadlocks
+* Newly created file descriptors are non-inheritable (PEP 446) (TESTING)
 
 * The marshal format has been made more compact and efficient
 

From pypy.commits at gmail.com  Sat Aug 27 08:57:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 05:57:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: Marshal version 3
Message-ID: <57c18e2e.17a61c0a.34ddb.3e77@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86598:79a816d8f40e
Date: 2016-08-27 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/79a816d8f40e/

Log:	Marshal version 3


From pypy.commits at gmail.com  Sat Aug 27 09:27:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 06:27:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: fix test
Message-ID: <57c1955f.448e1c0a.f4301.43a3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86599:8b236943d106
Date: 2016-08-27 15:27 +0200
http://bitbucket.org/pypy/pypy/changeset/8b236943d106/

Log:	fix test

diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -252,14 +252,15 @@
         assert posix.get_inheritable(f.fileno()) == False
         f.close()
 
-    def test_FileIO_fd_does_change_inheritable(self):
+    def test_FileIO_fd_does_not_change_inheritable(self):
         import _io, posix
         fd1, fd2 = posix.pipe()
         posix.set_inheritable(fd1, True)
+        posix.set_inheritable(fd2, False)
         f1 = _io.FileIO(fd1, 'r')
         f2 = _io.FileIO(fd2, 'w')
-        assert posix.get_inheritable(fd1) == False
-        assert posix.get_inheritable(fd2) == True
+        assert posix.get_inheritable(fd1) == True
+        assert posix.get_inheritable(fd2) == False
         f1.close()
         f2.close()
 

From pypy.commits at gmail.com  Sat Aug 27 10:19:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 07:19:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: Fixes in the test of fcntl
Message-ID: <57c1a17d.c398c20a.b1de1.21c4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86600:3a692066a13f
Date: 2016-08-27 16:19 +0200
http://bitbucket.org/pypy/pypy/changeset/3a692066a13f/

Log:	Fixes in the test of fcntl

diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -32,7 +32,7 @@
 
         f = open(self.tmp + "b", "w+")
 
-        fcntl.fcntl(f, 1, 0)
+        original = fcntl.fcntl(f, 1, 0)
         fcntl.fcntl(f, 1)
         fcntl.fcntl(F(int(f.fileno())), 1)
         raises(TypeError, fcntl.fcntl, "foo")
@@ -46,9 +46,16 @@
         raises(ValueError, fcntl.fcntl, -1, 1, 0)
         raises(ValueError, fcntl.fcntl, F(-1), 1, 0)
         raises(ValueError, fcntl.fcntl, F(int(-1)), 1, 0)
-        assert fcntl.fcntl(f, 1, 0) == 0
+        assert fcntl.fcntl(f, 1, 0) == original
         assert fcntl.fcntl(f, 2, "foo") == b"foo"
-        assert fcntl.fcntl(f, 2, memoryview(b"foo")) == b"foo"
+        assert fcntl.fcntl(f, 2, b"foo") == b"foo"
+
+        # This is supposed to work I think, but CPython 3.5 refuses it
+        # for reasons I don't understand:
+        #     >>> _testcapi.getargs_s_hash(memoryview(b"foo"))
+        #     TypeError: must be read-only bytes-like object, not memoryview
+        #
+        # assert fcntl.fcntl(f, 2, memoryview(b"foo")) == b"foo"
 
         try:
             os.O_LARGEFILE
@@ -202,7 +209,7 @@
         raises(TypeError, fcntl.ioctl, "foo")
         raises(TypeError, fcntl.ioctl, 0, "foo")
         #raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, float(0))
-        raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, 1, "foo")
+        raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, 1, "foo", "bar")
 
         child_pid, mfd = pty.fork()
         if child_pid == 0:
@@ -229,13 +236,13 @@
             assert res == 0
             assert buf.tostring() == expected
 
-            exc = raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, memoryview(b'abc'), False)
-            assert str(exc.value) == "ioctl requires a file or file descriptor, an integer and optionally an integer or buffer argument"
+            raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, (), False)
 
             res = fcntl.ioctl(mfd, TIOCGPGRP, buf, False)
             assert res == expected
 
-            raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, "\x00\x00", True)
+            # xxx this fails on CPython 3.5, that's a minor bug
+            #raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, "\x00\x00", True)
 
             res = fcntl.ioctl(mfd, TIOCGPGRP, "\x00\x00\x00\x00")
             assert res == expected

From pypy.commits at gmail.com  Sat Aug 27 10:33:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 07:33:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix for fda5486b3186 (sorry!): there
 was at least one untested place
Message-ID: <57c1a4ca.8628c20a.5d8b9.245f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86601:020068a372fd
Date: 2016-08-27 16:33 +0200
http://bitbucket.org/pypy/pypy/changeset/020068a372fd/

Log:	Fix for fda5486b3186 (sorry!): there was at least one untested place
	where a llexternal() function was called with more arguments than
	declared (here a space), which was previously ignored

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -977,9 +977,11 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
-    add_fork_hook('child', reinit_tls)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
+    add_fork_hook('child', _reinit_tls)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)

From pypy.commits at gmail.com  Sat Aug 27 10:35:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 07:35:22 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: More things to do later
Message-ID: <57c1a52a.d4e01c0a.e9e5b.5b93@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5685:430b20e00289
Date: 2016-08-27 16:35 +0200
http://bitbucket.org/pypy/extradoc/changeset/430b20e00289/

Log:	More things to do later

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -38,6 +38,11 @@
 
 * compare ``dir(posix)`` on py3.5 and cpython 3.5.
 
+* review all unwrap_spec() that are meant to pass booleans (now
+  with '=int').  Argument clinic turns these to PyObject_IsTrue(), i.e.
+  accepting any object whatsoever(?!), which is supposedly a feature
+  (see http://bugs.python.org/issue14705).
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------
diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -23,8 +23,17 @@
   threads concurrently.  It will make two stat objects and leak one of
   them.
 
-* (not a crasher) on modern Linux: if the first call in the process to
+
+Other bugs
+----------
+
+* on modern Linux: if the first call in the process to
   socketpair() ends in a EINVAL, then cpython will (possibly wrongly)
   assume it was caused by SOCK_CLOEXEC and not use SOCK_CLOEXEC at all
   in the future
 
+* fcntl.ioctl(x, y, buf, mutate_flag): mutate_flag is there for the case
+  of buf being a read-write buffer, which is then mutated in-place.
+  But if we call with a read-only buffer, mutate_flag is ignored (instead
+  of rejecting a True value)---ioctl(x, y, "foo", True) will not actually
+  mutate the string "foo", but the True is completely ignored.

From pypy.commits at gmail.com  Sat Aug 27 10:38:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 07:38:20 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: done
Message-ID: <57c1a5dc.d32d1c0a.a43ed.54c3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5686:8c4d7b3c02cc
Date: 2016-08-27 16:38 +0200
http://bitbucket.org/pypy/extradoc/changeset/8c4d7b3c02cc/

Log:	done

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -11,8 +11,6 @@
   We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
   Matti's opinion on that would be great!
 
-* arigo: Newly created file descriptors are non-inheritable (PEP 446)
-
 * arigo: New marshal format
 
 
@@ -70,7 +68,8 @@
 
 * The new os.scandir() function (POSIX-DONE, missing Win32)
 
-* Newly created file descriptors are non-inheritable (PEP 446) (TESTING)
+* Newly created file descriptors are non-inheritable (PEP 446) 
+  (POSIX-DONE, missing Win32)
 
 * The marshal format has been made more compact and efficient
 

From pypy.commits at gmail.com  Sat Aug 27 10:38:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 07:38:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: fix test
Message-ID: <57c1a5f4.497bc20a.5c897.1f48@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86602:89dd2d0333a8
Date: 2016-08-27 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/89dd2d0333a8/

Log:	fix test

diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -58,7 +58,7 @@
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
     expect(f, g, "ll_os.ll_os_open", ("/tmp/foobar", os.O_RDONLY, 0777), 77)
-    expect(f, g, "ll_os.ll_os_dup",  (77,), 78)
+    expect(f, g, "ll_os.ll_os_dup",  (77, True), 78)
     g.close()
     tail = f.read()
     f.close()
@@ -94,7 +94,7 @@
 
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
-    expect(f, g, "ll_os.ll_os_dup2",   (34, 56), None)
+    expect(f, g, "ll_os.ll_os_dup2",   (34, 56, True), None)
     expect(f, g, "ll_os.ll_os_access", ("spam", 77), True)
     g.close()
     tail = f.read()
@@ -134,7 +134,7 @@
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
     expect(f, g, "ll_time.ll_time_time", (), 3.141592)
-    expect(f, g, "ll_os.ll_os_dup", (3141,), 3)
+    expect(f, g, "ll_os.ll_os_dup", (3141, True), 3)
     g.close()
     tail = f.read()
     f.close()
@@ -149,7 +149,7 @@
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
     expect(f, g, "ll_os.ll_os_getcwd", (), "/tmp/foo/bar")
-    expect(f, g, "ll_os.ll_os_dup", (len("/tmp/foo/bar"),), 3)
+    expect(f, g, "ll_os.ll_os_dup", (len("/tmp/foo/bar"), True), 3)
     g.close()
     tail = f.read()
     f.close()

From pypy.commits at gmail.com  Sat Aug 27 10:38:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 07:38:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-noninherit: close branch, ready to merge
Message-ID: <57c1a5f6.041f1c0a.8d1b3.579f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-noninherit
Changeset: r86603:180b2ba69a38
Date: 2016-08-27 16:37 +0200
http://bitbucket.org/pypy/pypy/changeset/180b2ba69a38/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Sat Aug 27 10:38:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 07:38:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3.5-noninherit
Message-ID: <57c1a5f8.919a1c0a.bca81.626f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86604:6007963baadc
Date: 2016-08-27 16:37 +0200
http://bitbucket.org/pypy/pypy/changeset/6007963baadc/

Log:	hg merge py3.5-noninherit

	Newly created file descriptors are non-inheritable (PEP 446)

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -554,6 +554,9 @@
     def putwin(self, filep):
         # filestar = ffi.new("FILE *", filep)
         return _check_ERR(lib.putwin(self._win, filep), "putwin")
+        # XXX CPython 3.5 says: We have to simulate this by writing to
+        # a temporary FILE*, then reading back, then writing to the
+        # argument stream.
 
     def redrawln(self, beg, num):
         return _check_ERR(lib.wredrawln(self._win, beg, num), "redrawln")
@@ -704,6 +707,7 @@
 
 
 def getwin(filep):
+    # XXX CPython 3.5: there's logic to use a temp file instead
     return Window(_check_NULL(lib.getwin(filep)))
 
 
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -4,6 +4,7 @@
     OperationError, oefmt, wrap_oserror, wrap_oserror2)
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import rposix
 from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC, O_EXCL
 import sys, os, stat, errno
 from pypy.module._io.interp_iobase import W_RawIOBase, convert_size
@@ -29,6 +30,7 @@
 
 O_BINARY = getattr(os, "O_BINARY", 0)
 O_APPEND = getattr(os, "O_APPEND", 0)
+_open_inhcache = rposix.SetNonInheritableCache()
 
 def _bad_mode(space):
     raise oefmt(space.w_ValueError,
@@ -139,6 +141,7 @@
 
     @unwrap_spec(mode=str, closefd=int)
     def descr_init(self, space, w_name, mode='r', closefd=True, w_opener=None):
+        self._close(space)
         if space.isinstance_w(w_name, space.w_float):
             raise oefmt(space.w_TypeError,
                         "integer argument expected, got float")
@@ -153,6 +156,8 @@
                 raise oefmt(space.w_ValueError, "negative file descriptor")
 
         self.readable, self.writable, self.created, self.appending, flags = decode_mode(space, mode)
+        if rposix.O_CLOEXEC is not None:
+            flags |= rposix.O_CLOEXEC
 
         fd_is_own = False
         try:
@@ -171,8 +176,7 @@
                     raise oefmt(space.w_ValueError,
                                 "Cannot use closefd=False with file name")
 
-                from pypy.module.posix.interp_posix import (
-                    dispatch_filename, rposix)
+                from pypy.module.posix.interp_posix import dispatch_filename
                 try:
                     self.fd = dispatch_filename(rposix.open)(
                         space, w_name, flags, 0666)
@@ -181,6 +185,11 @@
                                         exception_name='w_IOError')
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    try:
+                        _open_inhcache.set_non_inheritable(self.fd)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
             else:
                 w_fd = space.call_function(w_opener, w_name, space.wrap(flags))
                 try:
@@ -192,6 +201,11 @@
                                 "expected integer from opener")
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    try:
+                        rposix.set_inheritable(self.fd, False)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
 
             self._dircheck(space, w_name)
             space.setattr(self, space.wrap("name"), w_name)
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -246,6 +246,33 @@
             assert f.mode == 'xb'
         raises(FileExistsError, _io.FileIO, filename, 'x')
 
+    def test_non_inheritable(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        assert posix.get_inheritable(f.fileno()) == False
+        f.close()
+
+    def test_FileIO_fd_does_not_change_inheritable(self):
+        import _io, posix
+        fd1, fd2 = posix.pipe()
+        posix.set_inheritable(fd1, True)
+        posix.set_inheritable(fd2, False)
+        f1 = _io.FileIO(fd1, 'r')
+        f2 = _io.FileIO(fd2, 'w')
+        assert posix.get_inheritable(fd1) == True
+        assert posix.get_inheritable(fd2) == False
+        f1.close()
+        f2.close()
+
+    def test_close_upon_reinit(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        fd1 = f.fileno()
+        f.__init__(self.tmpfile, 'w')
+        fd2 = f.fileno()
+        if fd1 != fd2:
+            raises(OSError, posix.close, fd1)
+
 
 def test_flush_at_exit():
     from pypy import conftest
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.c b/pypy/module/_posixsubprocess/_posixsubprocess.c
--- a/pypy/module/_posixsubprocess/_posixsubprocess.c
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.c
@@ -106,6 +106,30 @@
 }
 
 
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable);   /* rposix.py */
+
+static int
+make_inheritable(long *py_fds_to_keep, ssize_t num_fds_to_keep,
+                 int errpipe_write)
+{
+    long i;
+
+    for (i = 0; i < num_fds_to_keep; ++i) {
+        long fd = py_fds_to_keep[i];
+        if (fd == errpipe_write) {
+            /* errpipe_write is part of py_fds_to_keep. It must be closed at
+               exec(), but kept open in the child process until exec() is
+               called. */
+            continue;
+        }
+        if (rpy_set_inheritable((int)fd, 1) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+
 /* Close all file descriptors in the range start_fd inclusive to
  * end_fd exclusive except for those in py_fds_to_keep.  If the
  * range defined by [start_fd, end_fd) is large this will take a
@@ -329,6 +353,9 @@
     /* Buffer large enough to hold a hex integer.  We can't malloc. */
     char hex_errno[sizeof(saved_errno)*2+1];
 
+    if (make_inheritable(py_fds_to_keep, num_fds_to_keep, errpipe_write) < 0)
+        goto error;
+
     /* Close parent's pipe ends. */
     if (p2cwrite != -1) {
         POSIX_CALL(close(p2cwrite));
@@ -352,26 +379,25 @@
        dup2() removes the CLOEXEC flag but we must do it ourselves if dup2()
        would be a no-op (issue #10806). */
     if (p2cread == 0) {
-        int old = fcntl(p2cread, F_GETFD);
-        if (old != -1)
-            fcntl(p2cread, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (p2cread != -1) {
+        if (rpy_set_inheritable(p2cread, 1) < 0)
+            goto error;
+    }
+    else if (p2cread != -1)
         POSIX_CALL(dup2(p2cread, 0));  /* stdin */
+
+    if (c2pwrite == 1) {
+        if (rpy_set_inheritable(c2pwrite, 1) < 0)
+            goto error;
     }
-    if (c2pwrite == 1) {
-        int old = fcntl(c2pwrite, F_GETFD);
-        if (old != -1)
-            fcntl(c2pwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (c2pwrite != -1) {
+    else if (c2pwrite != -1)
         POSIX_CALL(dup2(c2pwrite, 1));  /* stdout */
+
+    if (errwrite == 2) {
+        if (rpy_set_inheritable(errwrite, 1) < 0)
+            goto error;
     }
-    if (errwrite == 2) {
-        int old = fcntl(errwrite, F_GETFD);
-        if (old != -1)
-            fcntl(errwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (errwrite != -1) {
+    else if (errwrite != -1)
         POSIX_CALL(dup2(errwrite, 2));  /* stderr */
-    }
 
     /* Close pipe fds.  Make sure we don't close the same fd more than */
     /* once, or standard fds. */
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.h b/pypy/module/_posixsubprocess/_posixsubprocess.h
--- a/pypy/module/_posixsubprocess/_posixsubprocess.h
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.h
@@ -1,3 +1,4 @@
+#include <unistd.h>    /* for ssize_t */
 #include "src/precommondefs.h"
 
 RPY_EXTERN void
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -5,6 +5,7 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib import rposix
 
 from pypy.interpreter.error import (
     OperationError, exception_from_saved_errno, oefmt, wrap_oserror)
@@ -36,6 +37,7 @@
     compile_extra.append("-DHAVE_SETSID")
 
 eci = eci.merge(
+    rposix.eci_inheritable,
     ExternalCompilationInfo(
         compile_extra=compile_extra))
 
diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py
--- a/pypy/module/_posixsubprocess/test/test_subprocess.py
+++ b/pypy/module/_posixsubprocess/test/test_subprocess.py
@@ -75,3 +75,18 @@
         n = 1
         raises(OverflowError, _posixsubprocess.fork_exec,
                1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17)
+
+    def test_pass_fds_make_inheritable(self):
+        import subprocess, posix
+
+        fd1, fd2 = posix.pipe()
+        assert posix.get_inheritable(fd1) is False
+        assert posix.get_inheritable(fd2) is False
+
+        subprocess.check_call(['/usr/bin/env', 'python', '-c',
+                               'import os;os.write(%d,b"K")' % fd2],
+                              close_fds=True, pass_fds=[fd2])
+        res = posix.read(fd1, 1)
+        assert res == b"K"
+        posix.close(fd1)
+        posix.close(fd2)
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -143,24 +143,11 @@
 @unwrap_spec(fd=int)
 def dup(space, fd):
     try:
-        newfd = rsocket.dup(fd)
+        newfd = rsocket.dup(fd, inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
     return space.wrap(newfd)
 
- at unwrap_spec(fd=int, family=int, type=int, proto=int)
-def fromfd(space, fd, family, type, proto=0):
-    """fromfd(fd, family, type[, proto]) -> socket object
-
-    Create a socket object from the given file descriptor.
-    The remaining arguments are the same as for socket().
-    """
-    try:
-        sock = rsocket.fromfd(fd, family, type, proto)
-    except SocketError as e:
-        raise converted_error(space, e)
-    return space.wrap(W_Socket(space, sock))
-
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
                       type  =rsocket.SOCK_STREAM,
@@ -173,7 +160,8 @@
     AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
     """
     try:
-        sock1, sock2 = rsocket.socketpair(family, type, proto)
+        sock1, sock2 = rsocket.socketpair(family, type, proto,
+                                          inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
     return space.newtuple([
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -177,7 +177,7 @@
                 sock = RSocket(family, type, proto,
                                fd=space.c_filedescriptor_w(w_fileno))
             else:
-                sock = RSocket(family, type, proto)
+                sock = RSocket(family, type, proto, inheritable=False)
             W_Socket.__init__(self, space, sock)
         except SocketError as e:
             raise converted_error(space, e)
@@ -228,7 +228,7 @@
         For IP sockets, the address info is a pair (hostaddr, port).
         """
         try:
-            fd, addr = self.sock.accept()
+            fd, addr = self.sock.accept(inheritable=False)
             return space.newtuple([space.wrap(fd),
                                    addr_as_object(addr, fd, space)])
         except SocketError as e:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -546,11 +546,15 @@
         s.ioctl(_socket.SIO_KEEPALIVE_VALS, (1, 100, 100))
 
     def test_dup(self):
-        import _socket as socket
+        import _socket as socket, posix
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.bind(('localhost', 0))
         fd = socket.dup(s.fileno())
         assert s.fileno() != fd
+        assert posix.get_inheritable(s.fileno()) is False
+        assert posix.get_inheritable(fd) is False
+        posix.close(fd)
+        s.close()
 
     def test_dup_error(self):
         import _socket
@@ -652,6 +656,26 @@
         assert len(w) == 1, [str(warning) for warning in w]
         assert r in str(w[0])
 
+    def test_invalid_fd(self):
+        import _socket
+        raises(ValueError, _socket.socket, fileno=-1)
+
+    def test_socket_non_inheritable(self):
+        import _socket, posix
+        s1 = _socket.socket()
+        assert posix.get_inheritable(s1.fileno()) is False
+        s1.close()
+
+    def test_socketpair_non_inheritable(self):
+        import _socket, posix
+        if not hasattr(_socket, 'socketpair'):
+            skip("no socketpair")
+        s1, s2 = _socket.socketpair()
+        assert posix.get_inheritable(s1.fileno()) is False
+        assert posix.get_inheritable(s2.fileno()) is False
+        s1.close()
+        s2.close()
+
 
 class AppTestNetlink:
     def setup_class(cls):
@@ -830,6 +854,16 @@
         assert cli.family == socket.AF_INET
 
 
+    def test_accept_non_inheritable(self):
+        import _socket, posix
+        cli = _socket.socket()
+        cli.connect(self.serv.getsockname())
+        fileno, addr = self.serv._accept()
+        assert posix.get_inheritable(fileno) is False
+        posix.close(fileno)
+        cli.close()
+
+
 class AppTestErrno:
     spaceconfig = {'usemodules': ['_socket']}
 
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -32,7 +32,7 @@
 
         f = open(self.tmp + "b", "w+")
 
-        fcntl.fcntl(f, 1, 0)
+        original = fcntl.fcntl(f, 1, 0)
         fcntl.fcntl(f, 1)
         fcntl.fcntl(F(int(f.fileno())), 1)
         raises(TypeError, fcntl.fcntl, "foo")
@@ -46,9 +46,16 @@
         raises(ValueError, fcntl.fcntl, -1, 1, 0)
         raises(ValueError, fcntl.fcntl, F(-1), 1, 0)
         raises(ValueError, fcntl.fcntl, F(int(-1)), 1, 0)
-        assert fcntl.fcntl(f, 1, 0) == 0
+        assert fcntl.fcntl(f, 1, 0) == original
         assert fcntl.fcntl(f, 2, "foo") == b"foo"
-        assert fcntl.fcntl(f, 2, memoryview(b"foo")) == b"foo"
+        assert fcntl.fcntl(f, 2, b"foo") == b"foo"
+
+        # This is supposed to work I think, but CPython 3.5 refuses it
+        # for reasons I don't understand:
+        #     >>> _testcapi.getargs_s_hash(memoryview(b"foo"))
+        #     TypeError: must be read-only bytes-like object, not memoryview
+        #
+        # assert fcntl.fcntl(f, 2, memoryview(b"foo")) == b"foo"
 
         try:
             os.O_LARGEFILE
@@ -202,7 +209,7 @@
         raises(TypeError, fcntl.ioctl, "foo")
         raises(TypeError, fcntl.ioctl, 0, "foo")
         #raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, float(0))
-        raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, 1, "foo")
+        raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, 1, "foo", "bar")
 
         child_pid, mfd = pty.fork()
         if child_pid == 0:
@@ -229,13 +236,13 @@
             assert res == 0
             assert buf.tostring() == expected
 
-            exc = raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, memoryview(b'abc'), False)
-            assert str(exc.value) == "ioctl requires a file or file descriptor, an integer and optionally an integer or buffer argument"
+            raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, (), False)
 
             res = fcntl.ioctl(mfd, TIOCGPGRP, buf, False)
             assert res == expected
 
-            raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, "\x00\x00", True)
+            # xxx this fails on CPython 3.5, that's a minor bug
+            #raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, "\x00\x00", True)
 
             res = fcntl.ioctl(mfd, TIOCGPGRP, "\x00\x00\x00\x00")
             assert res == expected
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -78,6 +78,8 @@
         'get_terminal_size': 'interp_posix.get_terminal_size',
 
         'scandir': 'interp_scandir.scandir',
+        'get_inheritable': 'interp_posix.get_inheritable',
+        'set_inheritable': 'interp_posix.set_inheritable',
     }
 
     if hasattr(os, 'chown'):
@@ -195,6 +197,9 @@
     interpleveldefs['_have_functions'] = (
         'space.newlist([space.wrap(x) for x in interp_posix.have_functions])')
 
+    if rposix.HAVE_PIPE2:
+        interpleveldefs['pipe2'] = 'interp_posix.pipe2'
+
     def startup(self, space):
         from pypy.module.posix import interp_posix
         from pypy.module.imp import importing
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -211,6 +211,8 @@
             space.w_NotImplementedError,
             "%s: %s unavailable on this platform", funcname, arg)
 
+_open_inhcache = rposix.SetNonInheritableCache()
+
 @unwrap_spec(flags=c_int, mode=c_int, dir_fd=DirFD(rposix.HAVE_OPENAT))
 def open(space, w_path, flags, mode=0777,
          __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
@@ -222,12 +224,15 @@
   and path should be relative; path will then be relative to that directory.
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
+    if rposix.O_CLOEXEC is not None:
+        flags |= rposix.O_CLOEXEC
     try:
         if rposix.HAVE_OPENAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             fd = rposix.openat(path, flags, mode, dir_fd)
         else:
             fd = dispatch_filename(rposix.open)(space, w_path, flags, mode)
+        _open_inhcache.set_non_inheritable(fd)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     return space.wrap(fd)
@@ -538,17 +543,17 @@
     """Create a copy of the file descriptor.  Return the new file
 descriptor."""
     try:
-        newfd = os.dup(fd)
+        newfd = rposix.dup(fd, inheritable=False)
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
         return space.wrap(newfd)
 
- at unwrap_spec(old_fd=c_int, new_fd=c_int)
-def dup2(space, old_fd, new_fd):
+ at unwrap_spec(old_fd=c_int, new_fd=c_int, inheritable=int)
+def dup2(space, old_fd, new_fd, inheritable=1):
     """Duplicate a file descriptor."""
     try:
-        os.dup2(old_fd, new_fd)
+        rposix.dup2(old_fd, new_fd, inheritable)
     except OSError as e:
         raise wrap_oserror(space, e)
 
@@ -891,15 +896,38 @@
             result_w[i] = space.fsdecode(w_bytes)
     return space.newlist(result_w)
 
+ at unwrap_spec(fd=c_int)
+def get_inheritable(space, fd):
+    try:
+        return space.wrap(rposix.get_inheritable(fd))
+    except OSError as e:
+        raise wrap_oserror(space, e)
+
+ at unwrap_spec(fd=c_int, inheritable=int)
+def set_inheritable(space, fd, inheritable):
+    try:
+        rposix.set_inheritable(fd, inheritable)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+
+_pipe_inhcache = rposix.SetNonInheritableCache()
+
 def pipe(space):
     "Create a pipe.  Returns (read_end, write_end)."
     try:
-        fd1, fd2 = os.pipe()
+        fd1, fd2 = rposix.pipe(rposix.O_CLOEXEC or 0)
+        _pipe_inhcache.set_non_inheritable(fd1)
+        _pipe_inhcache.set_non_inheritable(fd2)
     except OSError as e:
         raise wrap_oserror(space, e)
-    # XXX later, use rposix.pipe2() if available!
-    rposix.set_inheritable(fd1, False)
-    rposix.set_inheritable(fd2, False)
+    return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
+
+ at unwrap_spec(flags=c_int)
+def pipe2(space, flags):
+    try:
+        fd1, fd2 = rposix.pipe2(flags)
+    except OSError as e:
+        raise wrap_oserror(space, e)
     return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),
@@ -1238,6 +1266,8 @@
     "Open a pseudo-terminal, returning open fd's for both master and slave end."
     try:
         master_fd, slave_fd = os.openpty()
+        rposix.set_inheritable(master_fd, False)
+        rposix.set_inheritable(slave_fd, False)
     except OSError as e:
         raise wrap_oserror(space, e)
     return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)])
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -106,6 +106,7 @@
         posix = self.posix
         fd = posix.open(path, posix.O_RDONLY, 0o777)
         fd2 = posix.dup(fd)
+        assert posix.get_inheritable(fd2) == False
         assert not posix.isatty(fd2)
         s = posix.read(fd, 1)
         assert s == b't'
@@ -398,6 +399,16 @@
             os.write(slave_fd, b'x\n')
             data = os.read(master_fd, 100)
             assert data.startswith(b'x')
+            os.close(master_fd)
+            os.close(slave_fd)
+
+        def test_openpty_non_inheritable(self):
+            os = self.posix
+            master_fd, slave_fd = os.openpty()
+            assert os.get_inheritable(master_fd) == False
+            assert os.get_inheritable(slave_fd) == False
+            os.close(master_fd)
+            os.close(slave_fd)
 
     if hasattr(__import__(os.name), "forkpty"):
         def test_forkpty(self):
@@ -1077,6 +1088,52 @@
         x = f.read(1)
         assert x == 'e'
 
+    def test_pipe_inheritable(self):
+        fd1, fd2 = self.posix.pipe()
+        assert self.posix.get_inheritable(fd1) == False
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_pipe2(self):
+        if not hasattr(self.posix, 'pipe2'):
+            skip("no pipe2")
+        fd1, fd2 = self.posix.pipe2(0)
+        assert self.posix.get_inheritable(fd1) == True
+        assert self.posix.get_inheritable(fd2) == True
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_O_CLOEXEC(self):
+        if not hasattr(self.posix, 'pipe2'):
+            skip("no pipe2")
+        if not hasattr(self.posix, 'O_CLOEXEC'):
+            skip("no O_CLOEXEC")
+        fd1, fd2 = self.posix.pipe2(self.posix.O_CLOEXEC)
+        assert self.posix.get_inheritable(fd1) == False
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_dup2_inheritable(self):
+        fd1, fd2 = self.posix.pipe()
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.dup2(fd1, fd2)
+        assert self.posix.get_inheritable(fd2) == True
+        self.posix.dup2(fd1, fd2, False)
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.dup2(fd1, fd2, True)
+        assert self.posix.get_inheritable(fd2) == True
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_open_inheritable(self):
+        os = self.posix
+        fd = os.open(self.path2 + 'test_open_inheritable',
+                     os.O_RDWR | os.O_CREAT, 0o666)
+        assert os.get_inheritable(fd) == False
+        os.close(fd)
+
     def test_urandom(self):
         os = self.posix
         s = os.urandom(5)
diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py
--- a/pypy/module/select/interp_epoll.py
+++ b/pypy/module/select/interp_epoll.py
@@ -39,7 +39,8 @@
 for symbol in public_symbols:
     setattr(CConfig, symbol, rffi_platform.DefinedConstantInteger(symbol))
 
-for symbol in ["EPOLL_CTL_ADD", "EPOLL_CTL_MOD", "EPOLL_CTL_DEL"]:
+for symbol in ["EPOLL_CTL_ADD", "EPOLL_CTL_MOD", "EPOLL_CTL_DEL",
+               "EPOLL_CLOEXEC"]:
     setattr(CConfig, symbol, rffi_platform.ConstantInteger(symbol))
 
 cconfig = rffi_platform.configure(CConfig)
@@ -52,13 +53,14 @@
 EPOLL_CTL_ADD = cconfig["EPOLL_CTL_ADD"]
 EPOLL_CTL_MOD = cconfig["EPOLL_CTL_MOD"]
 EPOLL_CTL_DEL = cconfig["EPOLL_CTL_DEL"]
+EPOLL_CLOEXEC = cconfig["EPOLL_CLOEXEC"]
 
 DEF_REGISTER_EVENTMASK = (public_symbols["EPOLLIN"] |
                           public_symbols["EPOLLOUT"] |
                           public_symbols["EPOLLPRI"])
 
-epoll_create = rffi.llexternal(
-    "epoll_create", [rffi.INT], rffi.INT, compilation_info=eci,
+epoll_create1 = rffi.llexternal(
+    "epoll_create1", [rffi.INT], rffi.INT, compilation_info=eci,
     save_err=rffi.RFFI_SAVE_ERRNO
 )
 epoll_ctl = rffi.llexternal(
@@ -82,14 +84,12 @@
         self.epfd = epfd
         self.register_finalizer(space)
 
-    @unwrap_spec(sizehint=int)
-    def descr__new__(space, w_subtype, sizehint=-1):
-        if sizehint == -1:
-            sizehint = FD_SETSIZE - 1
-        elif sizehint < 0:
+    @unwrap_spec(sizehint=int, flags=int)
+    def descr__new__(space, w_subtype, sizehint=0, flags=0):
+        if sizehint < 0:     # 'sizehint' is otherwise ignored
             raise oefmt(space.w_ValueError,
                         "sizehint must be greater than zero, got %d", sizehint)
-        epfd = epoll_create(sizehint)
+        epfd = epoll_create1(flags | EPOLL_CLOEXEC)
         if epfd < 0:
             raise exception_from_saved_errno(space, space.w_IOError)
 
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -1,10 +1,11 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.error import exception_from_saved_errno
+from pypy.interpreter.error import exception_from_saved_errno, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.typedef import TypeDef, generic_new_descr, GetSetProperty
 from rpython.rlib._rsocket_rffi import socketclose_no_errno
 from rpython.rlib.rarithmetic import r_uint
+from rpython.rlib import rposix
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -115,6 +116,10 @@
         kqfd = syscall_kqueue()
         if kqfd < 0:
             raise exception_from_saved_errno(space, space.w_IOError)
+        try:
+            rposix.set_inheritable(kqfd, False)
+        except OSError as e:
+            raise wrap_oserror(space, e)
         return space.wrap(W_Kqueue(space, kqfd))
 
     @unwrap_spec(fd=int)
diff --git a/pypy/module/select/test/test_devpoll.py b/pypy/module/select/test/test_devpoll.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/select/test/test_devpoll.py
@@ -0,0 +1,4 @@
+# XXX
+
+# devpoll is not implemented, but if we do, make sure we test for
+# non-inheritable file descriptors
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -209,3 +209,10 @@
         ep = select.epoll()
         ep.close()
         ep.close()
+
+    def test_non_inheritable(self):
+        import select, posix
+
+        ep = select.epoll()
+        assert posix.get_inheritable(ep.fileno()) == False
+        ep.close()
diff --git a/pypy/module/select/test/test_kqueue.py b/pypy/module/select/test/test_kqueue.py
--- a/pypy/module/select/test/test_kqueue.py
+++ b/pypy/module/select/test/test_kqueue.py
@@ -186,3 +186,10 @@
         a.close()
         b.close()
         kq.close()
+
+    def test_non_inheritable(self):
+        import select, posix
+
+        kq = select.kqueue()
+        assert posix.get_inheritable(kq.fileno()) == False
+        kq.close()
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -176,6 +176,7 @@
 
 
 SOCK_DGRAM SOCK_RAW SOCK_RDM SOCK_SEQPACKET SOCK_STREAM
+SOCK_CLOEXEC
 
 SOL_SOCKET SOL_IPX SOL_AX25 SOL_ATALK SOL_NETROM SOL_ROSE
 
@@ -319,6 +320,8 @@
                                           [('p_proto', rffi.INT),
                                            ])
 
+CConfig.HAVE_ACCEPT4 = platform.Has('accept4')
+
 if _POSIX:
     CConfig.nfds_t = platform.SimpleType('nfds_t')
     CConfig.pollfd = platform.Struct('struct pollfd',
@@ -541,6 +544,12 @@
 socketaccept = external('accept', [socketfd_type, sockaddr_ptr,
                                    socklen_t_ptr], socketfd_type,
                         save_err=SAVE_ERR)
+HAVE_ACCEPT4 = cConfig.HAVE_ACCEPT4
+if HAVE_ACCEPT4:
+    socketaccept4 = external('accept4', [socketfd_type, sockaddr_ptr,
+                                         socklen_t_ptr, rffi.INT],
+                                        socketfd_type,
+                             save_err=SAVE_ERR)
 socketbind = external('bind', [socketfd_type, sockaddr_ptr, socklen_t],
                               rffi.INT, save_err=SAVE_ERR)
 socketlisten = external('listen', [socketfd_type, rffi.INT], rffi.INT,
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -372,15 +372,27 @@
         raise OSError(get_saved_errno(), '%s failed' % name)
     return result
 
+def _dup(fd, inheritable=True):
+    validate_fd(fd)
+    if inheritable:
+        res = c_dup(fd)
+    else:
+        res = c_dup_noninheritable(fd)
+    return res
+
 @replace_os_function('dup')
-def dup(fd):
-    validate_fd(fd)
-    return handle_posix_error('dup', c_dup(fd))
+def dup(fd, inheritable=True):
+    res = _dup(fd, inheritable)
+    return handle_posix_error('dup', res)
 
 @replace_os_function('dup2')
-def dup2(fd, newfd):
+def dup2(fd, newfd, inheritable=True):
     validate_fd(fd)
-    handle_posix_error('dup2', c_dup2(fd, newfd))
+    if inheritable:
+        res = c_dup2(fd, newfd)
+    else:
+        res = c_dup2_noninheritable(fd, newfd)
+    handle_posix_error('dup2', res)
 
 #___________________________________________________________________
 
@@ -1122,37 +1134,77 @@
     c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
                                                     rffi.INT],
                                 rffi.INT)
+    HAVE_PIPE2 = False
+    HAVE_DUP3 = False
+    O_CLOEXEC = None
 else:
     INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
     c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT,
                       save_err=rffi.RFFI_SAVE_ERRNO)
+    class CConfig:
+        _compilation_info_ = eci
+        HAVE_PIPE2 = rffi_platform.Has('pipe2')
+        HAVE_DUP3 = rffi_platform.Has('dup3')
+        O_CLOEXEC = rffi_platform.DefinedConstantInteger('O_CLOEXEC')
+    config = rffi_platform.configure(CConfig)
+    HAVE_PIPE2 = config['HAVE_PIPE2']
+    HAVE_DUP3 = config['HAVE_DUP3']
+    O_CLOEXEC = config['O_CLOEXEC']
+    if HAVE_PIPE2:
+        c_pipe2 = external('pipe2', [INT_ARRAY_P, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('pipe')
-def pipe():
+def pipe(flags=0):
+    # 'flags' might be ignored.  Check the result.
     if _WIN32:
+        # 'flags' ignored
         pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         try:
-            if not CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                raise WindowsError(rwin32.GetLastError_saved(),
-                                   "CreatePipe failed")
+            ok = CreatePipe(
+                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0)
             hread = rffi.cast(rffi.INTPTR_T, pread[0])
             hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
         finally:
             lltype.free(pwrite, flavor='raw')
             lltype.free(pread, flavor='raw')
-        fdread = c_open_osfhandle(hread, 0)
-        fdwrite = c_open_osfhandle(hwrite, 1)
+        if ok:
+            fdread = c_open_osfhandle(hread, 0)
+            fdwrite = c_open_osfhandle(hwrite, 1)
+            if fdread == -1 or fdwrite == -1:
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
+                ok = 0
+        if not ok:
+            raise WindowsError(rwin32.GetLastError_saved(),
+                               "CreatePipe failed")
         return (fdread, fdwrite)
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
-            handle_posix_error('pipe', c_pipe(filedes))
+            if HAVE_PIPE2 and _pipe2_syscall.attempt_syscall():
+                res = c_pipe2(filedes, flags)
+                if _pipe2_syscall.fallback(res):
+                    res = c_pipe(filedes)
+            else:
+                res = c_pipe(filedes)      # 'flags' ignored
+            handle_posix_error('pipe', res)
             return (widen(filedes[0]), widen(filedes[1]))
         finally:
             lltype.free(filedes, flavor='raw')
 
+def pipe2(flags):
+    # Only available if there is really a c_pipe2 function.
+    # No fallback to pipe() if we get ENOSYS.
+    filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
+    try:
+        res = c_pipe2(filedes, flags)
+        handle_posix_error('pipe2', res)
+        return (widen(filedes[0]), widen(filedes[1]))
+    finally:
+        lltype.free(filedes, flavor='raw')
+
 c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
                   save_err=rffi.RFFI_SAVE_ERRNO,)
 c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
@@ -2088,14 +2140,46 @@
 
 
 eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=["""
+    separate_module_sources=[r"""
+#include <errno.h>
+
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)
 {
-    /* XXX minimal impl. XXX */
-    int request = inheritable ? FIONCLEX : FIOCLEX;
-    return ioctl(fd, request, NULL);
+    static int ioctl_works = -1;
+    int flags;
+
+    if (ioctl_works != 0) {
+        int request = inheritable ? FIONCLEX : FIOCLEX;
+        int err = ioctl(fd, request, NULL);
+        if (!err) {
+            ioctl_works = 1;
+            return 0;
+        }
+
+        if (errno != ENOTTY && errno != EACCES) {
+            return -1;
+        }
+        else {
+            /* ENOTTY: The ioctl is declared but not supported by the
+               kernel.  EACCES: SELinux policy, this can be the case on
+               Android. */
+            ioctl_works = 0;
+        }
+        /* fallback to fcntl() if ioctl() does not work */
+    }
+
+    flags = fcntl(fd, F_GETFD);
+    if (flags < 0)
+        return -1;
+
+    if (inheritable)
+        flags &= ~FD_CLOEXEC;
+    else
+        flags |= FD_CLOEXEC;
+    return fcntl(fd, F_SETFD, flags);
 }
+
 RPY_EXTERN
 int rpy_get_inheritable(int fd)
 {
@@ -2104,8 +2188,64 @@
         return -1;
     return !(flags & FD_CLOEXEC);
 }
-    """],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+RPY_EXTERN
+int rpy_dup_noninheritable(int fd)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUPFD_CLOEXEC
+    return fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+    fd = dup(fd);
+    if (fd >= 0) {
+        if (rpy_set_inheritable(fd, 0) != 0) {
+            close(fd);
+            return -1;
+        }
+    }
+    return fd;
+#endif
+}
+
+RPY_EXTERN
+int rpy_dup2_noninheritable(int fd, int fd2)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUP2FD_CLOEXEC
+    return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
+
+#else
+# if %(HAVE_DUP3)d   /* HAVE_DUP3 */
+    static int dup3_works = -1;
+    if (dup3_works != 0) {
+        if (dup3(fd, fd2, O_CLOEXEC) >= 0)
+            return 0;
+        if (dup3_works == -1)
+            dup3_works = (errno != ENOSYS);
+        if (dup3_works)
+            return -1;
+    }
+# endif
+    if (dup2(fd, fd2) < 0)
+        return -1;
+    if (rpy_set_inheritable(fd2, 0) != 0) {
+        close(fd2);
+        return -1;
+    }
+    return 0;
+#endif
+}
+    """ % {'HAVE_DUP3': HAVE_DUP3}],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
 
 c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
@@ -2113,12 +2253,56 @@
 c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
                              compilation_info=eci_inheritable)
+c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                compilation_info=eci_inheritable)
+c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                 compilation_info=eci_inheritable)
 
 def set_inheritable(fd, inheritable):
-    error = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', error)
+    result = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', result)
 
 def get_inheritable(fd):
     res = c_get_inheritable(fd)
     res = handle_posix_error('get_inheritable', res)
     return res != 0
+
+class SetNonInheritableCache(object):
+    """Make one prebuilt instance of this for each path that creates
+    file descriptors, where you don't necessarily know if that function
+    returns inheritable or non-inheritable file descriptors.
+    """
+    _immutable_fields_ = ['cached_inheritable?']
+    cached_inheritable = -1    # -1 = don't know yet; 0 = off; 1 = on
+
+    def set_non_inheritable(self, fd):
+        if self.cached_inheritable == -1:
+            self.cached_inheritable = get_inheritable(fd)
+        if self.cached_inheritable == 1:
+            # 'fd' is inheritable; we must manually turn it off
+            set_inheritable(fd, False)
+
+    def _cleanup_(self):
+        self.cached_inheritable = -1
+
+class ENoSysCache(object):
+    """Cache whether a system call returns ENOSYS or not."""
+    _immutable_fields_ = ['cached_nosys?']
+    cached_nosys = -1      # -1 = don't know; 0 = no; 1 = yes, getting ENOSYS
+
+    def attempt_syscall(self):
+        return self.cached_nosys != 1
+
+    def fallback(self, res):
+        nosys = self.cached_nosys
+        if nosys == -1:
+            nosys = (res < 0 and get_saved_errno() == errno.ENOSYS)
+            self.cached_nosys = nosys
+        return nosys
+
+    def _cleanup_(self):
+        self.cached_nosys = -1
+
+_pipe2_syscall = ENoSysCache()
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -8,10 +8,11 @@
 # XXX this does not support yet the least common AF_xxx address families
 # supported by CPython.  See http://bugs.pypy.org/issue1942
 
+from errno import EINVAL
 from rpython.rlib import _rsocket_rffi as _c, jit, rgc
 from rpython.rlib.objectmodel import instantiate, keepalive_until_here
 from rpython.rlib.rarithmetic import intmask, r_uint
-from rpython.rlib import rthread
+from rpython.rlib import rthread, rposix
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rffi import sizeof, offsetof
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -522,12 +523,28 @@
     timeout = -1.0
 
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0,
-                 fd=_c.INVALID_SOCKET):
+                 fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            fd = _c.socket(family, type, proto)
-        if _c.invalid_socket(fd):
-            raise self.error_handler()
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socket() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                fd = _c.socket(family, type | SOCK_CLOEXEC, proto)
+                if fd < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise self.error_handler()
+            if _c.invalid_socket(fd):
+                fd = _c.socket(family, type, proto)
+                if _c.invalid_socket(fd):
+                    raise self.error_handler()
+                if not inheritable:
+                    sock_set_inheritable(fd, False)
         # PLAT RISCOS
         self.fd = fd
         self.family = family
@@ -630,20 +647,33 @@
         return addr, addr.addr_p, addrlen_p
 
     @jit.dont_look_inside
-    def accept(self):
+    def accept(self, inheritable=True):
         """Wait for an incoming connection.
         Return (new socket fd, client address)."""
         if self._select(False) == 1:
             raise SocketTimeout
         address, addr_p, addrlen_p = self._addrbuf()
         try:
-            newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+            remove_inheritable = not inheritable
+            if (not inheritable and SOCK_CLOEXEC is not None
+                    and _c.HAVE_ACCEPT4
+                    and _accept4_syscall.attempt_syscall()):
+                newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
+                                         SOCK_CLOEXEC)
+                if _accept4_syscall.fallback(newfd):
+                    newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+                else:
+                    remove_inheritable = False
+            else:
+                newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
             addrlen = addrlen_p[0]
         finally:
             lltype.free(addrlen_p, flavor='raw')
             address.unlock()
         if _c.invalid_socket(newfd):
             raise self.error_handler()
+        if remove_inheritable:
+            sock_set_inheritable(newfd, False)
         address.addrlen = rffi.cast(lltype.Signed, addrlen)
         return (newfd, address)
 
@@ -1032,6 +1062,12 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
+def sock_set_inheritable(fd, inheritable):
+    try:
+        rposix.set_inheritable(fd, inheritable)
+    except OSError as e:
+        raise CSocketError(e.errno)
+
 class SocketError(Exception):
     applevelerrcls = 'error'
     def __init__(self):
@@ -1090,7 +1126,7 @@
 
 if hasattr(_c, 'socketpair'):
     def socketpair(family=socketpair_default_family, type=SOCK_STREAM, proto=0,
-                   SocketClass=RSocket):
+                   SocketClass=RSocket, inheritable=True):
         """socketpair([family[, type[, proto]]]) -> (socket object, socket object)
 
         Create a pair of socket objects from the sockets returned by the platform
@@ -1099,17 +1135,42 @@
         AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
         """
         result = lltype.malloc(_c.socketpair_t, 2, flavor='raw')
-        res = _c.socketpair(family, type, proto, result)
-        if res < 0:
-            raise last_error()
-        fd0 = rffi.cast(lltype.Signed, result[0])
-        fd1 = rffi.cast(lltype.Signed, result[1])
-        lltype.free(result, flavor='raw')
+        try:
+            res = -1
+            remove_inheritable = not inheritable
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socketpair() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                res = _c.socketpair(family, type | SOCK_CLOEXEC,
+                                    proto, result)
+                if res < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise last_error()
+                else:
+                    remove_inheritable = False
+            #
+            if res < 0:
+                res = _c.socketpair(family, type, proto, result)
+                if res < 0:
+                    raise last_error()
+            fd0 = rffi.cast(lltype.Signed, result[0])
+            fd1 = rffi.cast(lltype.Signed, result[1])
+        finally:
+            lltype.free(result, flavor='raw')
+        if remove_inheritable:
+            sock_set_inheritable(fd0, False)
+            sock_set_inheritable(fd1, False)
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
 if _c.WIN32:
-    def dup(fd):
+    def dup(fd, inheritable=True):
         with lltype.scoped_alloc(_c.WSAPROTOCOL_INFO, zero=True) as info:
             if _c.WSADuplicateSocket(fd, rwin32.GetCurrentProcessId(), info):
                 raise last_error()
@@ -1120,15 +1181,16 @@
                 raise last_error()
             return result
 else:
-    def dup(fd):
-        fd = _c.dup(fd)
+    def dup(fd, inheritable=True):
+        fd = rposix._dup(fd, inheritable)
         if fd < 0:
             raise last_error()
         return fd
 
-def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
+def fromfd(fd, family, type, proto=0, SocketClass=RSocket, inheritable=True):
     # Dup the fd so it and the socket can be closed independently
-    return make_socket(dup(fd), family, type, proto, SocketClass)
+    fd = dup(fd, inheritable=inheritable)
+    return make_socket(fd, family, type, proto, SocketClass)
 
 def getdefaulttimeout():
     return defaults.timeout
@@ -1405,3 +1467,5 @@
     if timeout < 0.0:
         timeout = -1.0
     defaults.timeout = timeout
+
+_accept4_syscall = rposix.ENoSysCache()
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -589,3 +589,18 @@
     assert rposix.get_inheritable(fd1) == False
     os.close(fd1)
     os.close(fd2)
+
+def test_SetNonInheritableCache():
+    cache = rposix.SetNonInheritableCache()
+    fd1, fd2 = os.pipe()
+    assert rposix.get_inheritable(fd1) == True
+    assert rposix.get_inheritable(fd1) == True
+    assert cache.cached_inheritable == -1
+    cache.set_non_inheritable(fd1)
+    assert cache.cached_inheritable == 1
+    cache.set_non_inheritable(fd2)
+    assert cache.cached_inheritable == 1
+    assert rposix.get_inheritable(fd1) == False
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -119,6 +119,16 @@
     s1.close()
     s2.close()
 
+def test_socketpair_inheritable():
+    if sys.platform == "win32":
+        py.test.skip('No socketpair on Windows')
+    for inh in [False, True]:
+        s1, s2 = socketpair(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        assert rposix.get_inheritable(s2.fd) == inh
+        s1.close()
+        s2.close()
+
 def test_socketpair_recvinto_1():
     class Buffer:
         def setslice(self, start, string):
@@ -378,6 +388,12 @@
     s1.close()
     s2.close()
 
+def test_inheritable():
+    for inh in [False, True]:
+        s1 = RSocket(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        s1.close()
+
 def test_getaddrinfo_http():
     lst = getaddrinfo('localhost', 'http')
     assert isinstance(lst, list)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -237,8 +237,10 @@
                              " directly to a VOIDP argument")
     _oops._annspecialcase_ = 'specialize:memo'
 
+    nb_args = len(args)
     unrolling_arg_tps = unrolling_iterable(enumerate(args))
     def wrapper(*args):
+        assert len(args) == nb_args
         real_args = ()
         # XXX 'to_free' leaks if an allocation fails with MemoryError
         # and was not the first in this function
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -49,6 +49,7 @@
         eci = ExternalCompilationInfo(includes=['stuff.h'],
                                       include_dirs=[udir])
         z = llexternal('X', [Signed], Signed, compilation_info=eci)
+        py.test.raises(AssertionError, z, 8, 9)
 
         def f():
             return z(8)
diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -58,7 +58,7 @@
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
     expect(f, g, "ll_os.ll_os_open", ("/tmp/foobar", os.O_RDONLY, 0777), 77)
-    expect(f, g, "ll_os.ll_os_dup",  (77,), 78)
+    expect(f, g, "ll_os.ll_os_dup",  (77, True), 78)
     g.close()
     tail = f.read()
     f.close()
@@ -94,7 +94,7 @@
 
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
-    expect(f, g, "ll_os.ll_os_dup2",   (34, 56), None)
+    expect(f, g, "ll_os.ll_os_dup2",   (34, 56, True), None)
     expect(f, g, "ll_os.ll_os_access", ("spam", 77), True)
     g.close()
     tail = f.read()
@@ -134,7 +134,7 @@
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
     expect(f, g, "ll_time.ll_time_time", (), 3.141592)
-    expect(f, g, "ll_os.ll_os_dup", (3141,), 3)
+    expect(f, g, "ll_os.ll_os_dup", (3141, True), 3)
     g.close()
     tail = f.read()
     f.close()
@@ -149,7 +149,7 @@
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
     expect(f, g, "ll_os.ll_os_getcwd", (), "/tmp/foo/bar")
-    expect(f, g, "ll_os.ll_os_dup", (len("/tmp/foo/bar"),), 3)
+    expect(f, g, "ll_os.ll_os_dup", (len("/tmp/foo/bar"), True), 3)
     g.close()
     tail = f.read()
     f.close()

From pypy.commits at gmail.com  Sat Aug 27 10:50:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 07:50:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merger py3k
Message-ID: <57c1a8c2.cb7f1c0a.fa27d.5fc8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86606:1df27f12e886
Date: 2016-08-27 16:50 +0200
http://bitbucket.org/pypy/pypy/changeset/1df27f12e886/

Log:	hg merger py3k

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,219 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. 
+
+XXXXX MORE ???
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Support for memoryview attributes (format, itemsize, ...) which also
+    adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,157 +1,8 @@
-=========================
-What's new in PyPy2.7 5.3+
-=========================
+==========================
+What's new in PyPy2.7 5.4+
+==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 4176c6f63109
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
 
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
-
-
-.. branch: const-fold-we-are-jitted
-
-Reduce the size of the generated C code by constant-folding ``we_are_jitted``
-in non-jitcode.
-
-.. branch: memoryview-attributes
-
-Support for memoryview attributes (format, itemsize, ...).
-Extends the cpyext emulation layer.
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -0,0 +1,165 @@
+=========================
+What's new in PyPy2.7 5.4
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
+
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -356,9 +356,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -458,7 +464,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -610,7 +616,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -154,6 +154,7 @@
 
     def test_readValues(self):
         from winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -167,7 +168,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,7 +119,7 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 Py_CLEANUP_SUPPORTED
@@ -975,13 +975,15 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
     global py_fatalerror
     py_fatalerror = rffi.llexternal('%s_FatalError' % prefix,
                                     [CONST_STRING], lltype.Void,
                                     compilation_info=eci)
-    add_fork_hook('child', reinit_tls)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
+    add_fork_hook('child', _reinit_tls)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.4.1-alpha0"
+#define PYPY_VERSION_NUM  0x05040100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -1,3 +1,6 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
 #include <Python.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -10,7 +13,7 @@
 /* Structure defines a 1-dimensional strided array */
 typedef struct{
     int* arr;
-    long length;
+    Py_ssize_t length;
 } MyArray;
 
 /* initialize the array with integers 0...length */
@@ -61,13 +64,13 @@
 static int
 PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
 {
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
     // init may have already been called
     if (self->arr.arr != NULL) {
         deallocate_MyArray(&self->arr);
     }
 
-    int length = 0;
-    static char *kwlist[] = {"length", NULL};
     if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
         return -1;
 
@@ -103,16 +106,19 @@
 static int
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
 
-  PyMyArray* self = (PyMyArray*)obj;
   view->obj = (PyObject*)self;
   view->buf = (void*)self->arr.arr;
   view->len = self->arr.length * sizeof(int);
@@ -218,7 +224,6 @@
 #ifdef __GNUC__
 extern __attribute__((visibility("default")))
 #else
-extern __declspec(dllexport)
 #endif
 
 PyMODINIT_FUNC
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -77,4 +77,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -18,7 +18,8 @@
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER)
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -386,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -721,8 +724,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 2, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/module/test_lib_pypy/test_gdbm_extra.py b/pypy/module/test_lib_pypy/test_gdbm_extra.py
--- a/pypy/module/test_lib_pypy/test_gdbm_extra.py
+++ b/pypy/module/test_lib_pypy/test_gdbm_extra.py
@@ -15,3 +15,7 @@
     assert len(g) == 2
     del g['abc']
     assert len(g) == 1
+
+def test_unicode():
+    path = unicode(udir.join('test_gdm_unicode'))
+    g = gdbm.open(path, 'c')  # does not crash
diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import sys
 
 import os
 if os.name != 'posix':
@@ -47,6 +48,9 @@
     # minimal "does not crash" test
     x, y = resource.getrlimit(resource.RLIMIT_CPU)
     resource.setrlimit(resource.RLIMIT_CPU, (x, y))
-    x += 0.2
-    y += 0.3
-    resource.setrlimit(resource.RLIMIT_CPU, (x, y))    # truncated to ints
+    # sometimes, x and y are very large (more than 53 bits).
+    # for these huge values, int(float(x)) > x...
+    xf = x + (0.2 if x >= 0 else -0.2)
+    yf = y + (0.3 if y >= 0 else -0.3)
+    if int(xf) == x and int(yf) == y:
+        resource.setrlimit(resource.RLIMIT_CPU, (x, y))  # truncated to ints
diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -346,9 +346,16 @@
 
     def descr_hash(self, space):
         hashreal = _hash_float(space, self.realval)
-        hashimg = _hash_float(space, self.imagval)
-        combined = intmask(hashreal + HASH_IMAG * hashimg)
-        return space.newint(-2 if combined == -1 else combined)
+        hashimg = _hash_float(space, self.imagval)   # 0 if self.imagval == 0
+        h = intmask(hashreal + HASH_IMAG * hashimg)
+        h -= (h == -1)
+        return space.newint(h)
+
+    def descr_coerce(self, space, w_other):
+        w_other = self._to_complex(space, w_other)
+        if w_other is None:
+            return space.w_NotImplemented
+        return space.newtuple([self, w_other])
 
     def descr_format(self, space, w_format_spec):
         return newformat.run_formatter(space, w_format_spec, "format_complex",
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -399,7 +399,9 @@
     descr_str = func_with_new_name(descr_repr, 'descr_str')
 
     def descr_hash(self, space):
-        return space.wrap(_hash_float(space, self.floatval))
+        h = _hash_float(space, self.floatval)
+        h -= (h == -1)
+        return space.wrap(h)
 
     def descr_format(self, space, w_spec):
         return newformat.run_formatter(space, w_spec, "format_float", self)
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -991,7 +991,8 @@
             if index != INVALID:
                 attr = map.find_map_attr(attrname, index)
                 if attr is not None:
-                    # Note that if map.terminator is a DevolvedDictTerminator,
+                    # Note that if map.terminator is a DevolvedDictTerminator
+                    # or the class provides its own dict, not using mapdict, then:
                     # map.find_map_attr will always return None if index==DICT.
                     _fill_cache(pycode, nameindex, map, version_tag, attr.storageindex)
                     return w_obj._mapdict_read_storage(attr.storageindex)
@@ -1013,6 +1014,12 @@
 
 def LOOKUP_METHOD_mapdict_fill_cache_method(space, pycode, name, nameindex,
                                             w_obj, w_type, w_method):
+    # if the layout has a dict itself, then mapdict is not used for normal
+    # attributes. Then the cache won't be able to spot changes to the dict.
+    # Thus we don't cache. see test_bug_builtin_types_callmethod
+    if w_type.layout.typedef.hasdict:
+        return
+
     if w_method is None or isinstance(w_method, MutableCell):
         # don't cache the MutableCell XXX could be fixed
         return
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -99,6 +99,8 @@
             buf = SubBuffer(self.buf, start * itemsize, size * itemsize)
             return W_MemoryView(buf, self.format, itemsize)
         else:
+            # XXX needs to return a W_MemoryView with a NonContiguousSubBuffer
+            # maybe?  Need to check the cpyext requirements for that
             raise oefmt(space.w_NotImplementedError,
                         "XXX extended slicing")
 
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -671,3 +671,6 @@
         assert sign(z2.real) == -1
         assert sign(z2.real) == -1
         
+    def test_hash_minus_one(self):
+        assert hash(-1.0 + 0j) == -2
+        assert (-1.0 + 0j).__hash__() == -2
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -475,6 +475,10 @@
         s = '\U0001D7CF\U0001D7CE.4' # &#120783;&#120782;.4
         assert float(s) == 10.4
 
+    def test_hash_minus_one(self):
+        assert hash(-1.0) == -2
+        assert (-1.0).__hash__() == -2
+
 
 class AppTestFloatHex:
     spaceconfig = {
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1238,6 +1238,42 @@
         got = x.a
         assert got == 'd'
 
+    def test_bug_builtin_types_callmethod(self):
+        import sys
+        class D(type(sys)):
+            def mymethod(self):
+                return "mymethod"
+
+        def foobar():
+            return "foobar"
+
+        d = D('d')
+        res1 = d.mymethod()
+        d.mymethod = foobar
+        res2 = d.mymethod()
+        assert res1 == "mymethod"
+        assert res2 == "foobar"
+
+    def test_bug_builtin_types_load_attr(self):
+        import sys
+        class D(type(sys)):
+            def mymethod(self):
+                return "mymethod"
+
+        def foobar():
+            return "foobar"
+
+        d = D('d')
+        m = d.mymethod
+        res1 = m()
+        d.mymethod = foobar
+        m = d.mymethod
+        res2 = m()
+        assert res1 == "mymethod"
+        assert res2 == "foobar"
+
+
+
 class AppTestGlobalCaching(AppTestWithMapDict):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
 
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -0,0 +1,134 @@
+import pytest
+import sys
+from pypy.tool.pytest.objspace import gettestobjspace
+try:
+    from hypothesis import given, strategies, settings
+except ImportError:
+    pytest.skip("requires hypothesis")
+
+base_initargs = strategies.sampled_from([
+    ("object", (), False),
+    ("type(sys)", ("fake", ), True),
+    ("NewBase", (), True),
+    ("OldBase", (), False),
+    ("object, OldBase", (), False),
+    ("type(sys), OldBase", ("fake", ), True),
+    ])
+
+attrnames = strategies.sampled_from(["a", "b", "c"])
+
+ at strategies.composite
+def make_code(draw):
+    # now here we can do this kind of thing:
+    baseclass, initargs, hasdict = draw(base_initargs)
+    # and with arbitrary strategies
+
+    def class_attr():
+        what = draw(strategies.sampled_from(["value", "method", "property"]))
+        if what == "value":
+            val = draw(strategies.integers())
+            return val, str(val)
+        if what == "method":
+            val = draw(strategies.integers())
+            return (lambda self, val=val: val,
+                    "lambda self: %d" % val)
+        if what == "property":
+            val = draw(strategies.integers())
+            return (property(lambda self, val=val: val,
+                             lambda self, val: None,
+                             lambda self: None),
+                    "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+    code = ["import sys", "class OldBase:pass", "class NewBase(object):pass", "class A(%s):" % baseclass]
+    dct = {}
+    if draw(strategies.booleans()):
+        slots = draw(strategies.lists(attrnames))
+        if not hasdict and draw(strategies.booleans()):
+            slots.append("__dict__")
+        dct["__slots__"] = slots
+        code.append("    __slots__ = %s" % (slots, ))
+    for name in ["a", "b", "c"]:
+        if not draw(strategies.booleans()):
+            continue
+        dct[name], codeval = class_attr()
+        code.append("    %s = %s" % (name, codeval))
+    class OldBase: pass
+    class NewBase(object): pass
+    evaldct = {'OldBase': OldBase, 'NewBase': NewBase}
+    if baseclass == 'OldBase':
+        metaclass = type(OldBase)
+    else:
+        metaclass = type
+    cls = metaclass("A", eval(baseclass+',', globals(), evaldct), dct)
+    inst = cls(*initargs)
+    code.append("    pass")
+    code.append("a = A(*%s)" % (initargs, ))
+    for attr in draw(strategies.lists(attrnames, min_size=1)):
+        op = draw(strategies.sampled_from(["read", "read", "read",
+                      "write", "writemeth", "writeclass", "writebase",
+                      "del", "delclass"]))
+        if op == "read":
+            try:
+                res = getattr(inst, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s')" % (attr, ))
+            else:
+                if callable(res):
+                    code.append("assert a.%s() == %s" % (attr, res()))
+                else:
+                    code.append("assert a.%s == %s" % (attr, res))
+        elif op == "write":
+            val = draw(strategies.integers())
+            try:
+                setattr(inst, attr, val)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s=%s')" % (attr, val))
+            else:
+                code.append("a.%s = %s" % (attr, val))
+        elif op == "writemeth":
+            val = draw(strategies.integers())
+            try:
+                setattr(inst, attr, lambda val=val: val)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s=0')" % (attr, ))
+            else:
+                code.append("a.%s = lambda : %s" % (attr, val))
+        elif op == "writeclass":
+            val, codeval = class_attr()
+            setattr(cls, attr, val)
+            code.append("A.%s = %s" % (attr, codeval))
+        elif op == "writebase":
+            val, codeval = class_attr()
+            setattr(OldBase, attr, val)
+            setattr(NewBase, attr, val)
+            code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))
+        elif op == "del":
+            try:
+                delattr(inst, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'del a.%s')" % (attr, ))
+            else:
+                code.append("del a.%s" % (attr, ))
+        elif op == "delclass":
+            try:
+                delattr(cls, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'del A.%s')" % (attr, ))
+            else:
+                code.append("del A.%s" % (attr, ))
+    return "\n    ".join(code)
+
+
+ at given(make_code())
+#@settings(max_examples=5000)
+def test_random_attrs(code):
+    try:
+        import __pypy__
+    except ImportError:
+        pass
+    else:
+        pytest.skip("makes no sense under pypy!")
+    space = gettestobjspace()
+    print code
+    exec "if 1:\n    " + code
+    space.appexec([], "():\n    " + code)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1064,6 +1064,8 @@
         else:
             assert mc.get_relative_pos() <= 13
         mc.copy_to_raw_memory(oldadr)
+        # log the redirection of the call_assembler_* operation
+        jl.redirect_assembler(oldlooptoken, newlooptoken, target)
 
     def dump(self, text):
         if not self.verbose:
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 2
+JITLOG_VERSION = 3
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -244,6 +244,7 @@
     ('COMMON_PREFIX',),
     ('ABORT_TRACE',),
     ('SOURCE_CODE',),
+    ('REDIRECT_ASSEMBLER',),
 ]
 
 start = 0x11
@@ -309,6 +310,17 @@
     content = ''.join(list)
     jitlog_write_marked(content, len(content))
 
+def redirect_assembler(oldtoken, newtoken, target):
+    if not jitlog_enabled():
+        return
+    descr_nmr = compute_unique_id(oldtoken)
+    new_descr_nmr = compute_unique_id(newtoken)
+    list = [MARK_REDIRECT_ASSEMBLER, encode_le_addr(descr_nmr),
+            encode_le_addr(new_descr_nmr), encode_le_addr(target)]
+    content = ''.join(list)
+    jitlog_write_marked(content, len(content))
+
+
 class JitLogger(object):
     def __init__(self, cpu=None):
         self.cpu = cpu
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -6,6 +6,15 @@
 from rpython.jit.backend.model import AbstractCPU
 from rpython.jit.metainterp.history import ConstInt, ConstPtr
 from rpython.rlib.rjitlog import rjitlog as jl
+from rpython.jit.metainterp.history import AbstractDescr
+from rpython.rlib.objectmodel import compute_unique_id
+
+class FakeCallAssemblerLoopToken(AbstractDescr):
+    def __init__(self, target):
+        self._ll_function_addr = target
+
+    def repr_of_descr(self):
+        return 'looptoken'
 
 class FakeLog(object):
     def __init__(self):
@@ -109,3 +118,31 @@
         with py.test.raises(AssertionError):
             jl.commonprefix(None,None)
 
+    def test_redirect_assembler(self, tmpdir):
+        looptoken = FakeCallAssemblerLoopToken(0x0)
+        newlooptoken = FakeCallAssemblerLoopToken(0x1234)
+        #
+        logger = jl.JitLogger()
+        file = tmpdir.join('binary_file')
+        file.ensure()
+        fd = file.open('wb')
+        jl.jitlog_init(fd.fileno())
+        logger.start_new_trace(self.make_metainterp_sd(), jd_name='jdname')
+        log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
+        op = ResOperation(rop.CALL_ASSEMBLER_I, [], descr=looptoken)
+        log_trace.write([], [op])
+        jl.redirect_assembler(looptoken, newlooptoken, 0x1234)
+        #the next line will close 'fd', instead of logger.finish()
+        fd.close()
+        binary = file.read()
+        opnum = jl.encode_le_16bit(rop.CALL_ASSEMBLER_I)
+        id_looptoken = compute_unique_id(looptoken)
+        new_id_looptoken = compute_unique_id(newlooptoken)
+        end = jl.MARK_RESOP_DESCR + opnum + jl.encode_str('i0,looptoken') + \
+              jl.encode_le_addr(id_looptoken) + jl.encode_str('') + \
+              jl.MARK_REDIRECT_ASSEMBLER + \
+              jl.encode_le_addr(id_looptoken) + \
+              jl.encode_le_addr(new_id_looptoken) + \
+              jl.encode_le_addr(newlooptoken._ll_function_addr)
+        assert binary.endswith(end)
+        

From pypy.commits at gmail.com  Sat Aug 27 10:50:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 07:50:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c1a8c0.031dc20a.f5e91.2977@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86605:471b3d7456a2
Date: 2016-08-27 16:49 +0200
http://bitbucket.org/pypy/pypy/changeset/471b3d7456a2/

Log:	hg merge default

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,219 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. 
+
+XXXXX MORE ???
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Support for memoryview attributes (format, itemsize, ...) which also
+    adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,157 +1,8 @@
-=========================
-What's new in PyPy2.7 5.3+
-=========================
+==========================
+What's new in PyPy2.7 5.4+
+==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 4176c6f63109
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
 
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
-
-
-.. branch: const-fold-we-are-jitted
-
-Reduce the size of the generated C code by constant-folding ``we_are_jitted``
-in non-jitcode.
-
-.. branch: memoryview-attributes
-
-Support for memoryview attributes (format, itemsize, ...).
-Extends the cpyext emulation layer.
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -0,0 +1,165 @@
+=========================
+What's new in PyPy2.7 5.4
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
+
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -356,9 +356,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -458,7 +464,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -610,7 +616,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -154,6 +154,7 @@
 
     def test_readValues(self):
         from winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -167,7 +168,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,7 +119,7 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 Py_CLEANUP_SUPPORTED
@@ -975,13 +975,15 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
     global py_fatalerror
     py_fatalerror = rffi.llexternal('%s_FatalError' % prefix,
                                     [CONST_STRING], lltype.Void,
                                     compilation_info=eci)
-    add_fork_hook('child', reinit_tls)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
+    add_fork_hook('child', _reinit_tls)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.4.1-alpha0"
+#define PYPY_VERSION_NUM  0x05040100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -1,3 +1,6 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
 #include <Python.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -10,7 +13,7 @@
 /* Structure defines a 1-dimensional strided array */
 typedef struct{
     int* arr;
-    long length;
+    Py_ssize_t length;
 } MyArray;
 
 /* initialize the array with integers 0...length */
@@ -61,13 +64,13 @@
 static int
 PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
 {
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
     // init may have already been called
     if (self->arr.arr != NULL) {
         deallocate_MyArray(&self->arr);
     }
 
-    int length = 0;
-    static char *kwlist[] = {"length", NULL};
     if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
         return -1;
 
@@ -103,16 +106,19 @@
 static int
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
 
-  PyMyArray* self = (PyMyArray*)obj;
   view->obj = (PyObject*)self;
   view->buf = (void*)self->arr.arr;
   view->len = self->arr.length * sizeof(int);
@@ -218,7 +224,6 @@
 #ifdef __GNUC__
 extern __attribute__((visibility("default")))
 #else
-extern __declspec(dllexport)
 #endif
 
 PyMODINIT_FUNC
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -77,4 +77,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -18,7 +18,8 @@
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER)
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -386,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -721,8 +724,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -398,6 +398,8 @@
             os.write(slave_fd, b'x\n')
             data = os.read(master_fd, 100)
             assert data.startswith(b'x')
+            os.close(master_fd)
+            os.close(slave_fd)
 
     if hasattr(__import__(os.name), "forkpty"):
         def test_forkpty(self):
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 2, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/module/test_lib_pypy/test_gdbm_extra.py b/pypy/module/test_lib_pypy/test_gdbm_extra.py
--- a/pypy/module/test_lib_pypy/test_gdbm_extra.py
+++ b/pypy/module/test_lib_pypy/test_gdbm_extra.py
@@ -15,3 +15,7 @@
     assert len(g) == 2
     del g['abc']
     assert len(g) == 1
+
+def test_unicode():
+    path = unicode(udir.join('test_gdm_unicode'))
+    g = gdbm.open(path, 'c')  # does not crash
diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import sys
 
 import os
 if os.name != 'posix':
@@ -47,6 +48,9 @@
     # minimal "does not crash" test
     x, y = resource.getrlimit(resource.RLIMIT_CPU)
     resource.setrlimit(resource.RLIMIT_CPU, (x, y))
-    x += 0.2
-    y += 0.3
-    resource.setrlimit(resource.RLIMIT_CPU, (x, y))    # truncated to ints
+    # sometimes, x and y are very large (more than 53 bits).
+    # for these huge values, int(float(x)) > x...
+    xf = x + (0.2 if x >= 0 else -0.2)
+    yf = y + (0.3 if y >= 0 else -0.3)
+    if int(xf) == x and int(yf) == y:
+        resource.setrlimit(resource.RLIMIT_CPU, (x, y))  # truncated to ints
diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -346,9 +346,16 @@
 
     def descr_hash(self, space):
         hashreal = _hash_float(space, self.realval)
-        hashimg = _hash_float(space, self.imagval)
-        combined = intmask(hashreal + HASH_IMAG * hashimg)
-        return space.newint(-2 if combined == -1 else combined)
+        hashimg = _hash_float(space, self.imagval)   # 0 if self.imagval == 0
+        h = intmask(hashreal + HASH_IMAG * hashimg)
+        h -= (h == -1)
+        return space.newint(h)
+
+    def descr_coerce(self, space, w_other):
+        w_other = self._to_complex(space, w_other)
+        if w_other is None:
+            return space.w_NotImplemented
+        return space.newtuple([self, w_other])
 
     def descr_format(self, space, w_format_spec):
         return newformat.run_formatter(space, w_format_spec, "format_complex",
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -399,7 +399,9 @@
     descr_str = func_with_new_name(descr_repr, 'descr_str')
 
     def descr_hash(self, space):
-        return space.wrap(_hash_float(space, self.floatval))
+        h = _hash_float(space, self.floatval)
+        h -= (h == -1)
+        return space.wrap(h)
 
     def descr_format(self, space, w_spec):
         return newformat.run_formatter(space, w_spec, "format_float", self)
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -991,7 +991,8 @@
             if index != INVALID:
                 attr = map.find_map_attr(attrname, index)
                 if attr is not None:
-                    # Note that if map.terminator is a DevolvedDictTerminator,
+                    # Note that if map.terminator is a DevolvedDictTerminator
+                    # or the class provides its own dict, not using mapdict, then:
                     # map.find_map_attr will always return None if index==DICT.
                     _fill_cache(pycode, nameindex, map, version_tag, attr.storageindex)
                     return w_obj._mapdict_read_storage(attr.storageindex)
@@ -1013,6 +1014,12 @@
 
 def LOOKUP_METHOD_mapdict_fill_cache_method(space, pycode, name, nameindex,
                                             w_obj, w_type, w_method):
+    # if the layout has a dict itself, then mapdict is not used for normal
+    # attributes. Then the cache won't be able to spot changes to the dict.
+    # Thus we don't cache. see test_bug_builtin_types_callmethod
+    if w_type.layout.typedef.hasdict:
+        return
+
     if w_method is None or isinstance(w_method, MutableCell):
         # don't cache the MutableCell XXX could be fixed
         return
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -671,3 +671,6 @@
         assert sign(z2.real) == -1
         assert sign(z2.real) == -1
         
+    def test_hash_minus_one(self):
+        assert hash(-1.0 + 0j) == -2
+        assert (-1.0 + 0j).__hash__() == -2
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -475,6 +475,10 @@
         s = '\U0001D7CF\U0001D7CE.4' # &#120783;&#120782;.4
         assert float(s) == 10.4
 
+    def test_hash_minus_one(self):
+        assert hash(-1.0) == -2
+        assert (-1.0).__hash__() == -2
+
 
 class AppTestFloatHex:
     spaceconfig = {
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1238,6 +1238,42 @@
         got = x.a
         assert got == 'd'
 
+    def test_bug_builtin_types_callmethod(self):
+        import sys
+        class D(type(sys)):
+            def mymethod(self):
+                return "mymethod"
+
+        def foobar():
+            return "foobar"
+
+        d = D('d')
+        res1 = d.mymethod()
+        d.mymethod = foobar
+        res2 = d.mymethod()
+        assert res1 == "mymethod"
+        assert res2 == "foobar"
+
+    def test_bug_builtin_types_load_attr(self):
+        import sys
+        class D(type(sys)):
+            def mymethod(self):
+                return "mymethod"
+
+        def foobar():
+            return "foobar"
+
+        d = D('d')
+        m = d.mymethod
+        res1 = m()
+        d.mymethod = foobar
+        m = d.mymethod
+        res2 = m()
+        assert res1 == "mymethod"
+        assert res2 == "foobar"
+
+
+
 class AppTestGlobalCaching(AppTestWithMapDict):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
 
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -0,0 +1,134 @@
+import pytest
+import sys
+from pypy.tool.pytest.objspace import gettestobjspace
+try:
+    from hypothesis import given, strategies, settings
+except ImportError:
+    pytest.skip("requires hypothesis")
+
+base_initargs = strategies.sampled_from([
+    ("object", (), False),
+    ("type(sys)", ("fake", ), True),
+    ("NewBase", (), True),
+    ("OldBase", (), False),
+    ("object, OldBase", (), False),
+    ("type(sys), OldBase", ("fake", ), True),
+    ])
+
+attrnames = strategies.sampled_from(["a", "b", "c"])
+
+ at strategies.composite
+def make_code(draw):
+    # now here we can do this kind of thing:
+    baseclass, initargs, hasdict = draw(base_initargs)
+    # and with arbitrary strategies
+
+    def class_attr():
+        what = draw(strategies.sampled_from(["value", "method", "property"]))
+        if what == "value":
+            val = draw(strategies.integers())
+            return val, str(val)
+        if what == "method":
+            val = draw(strategies.integers())
+            return (lambda self, val=val: val,
+                    "lambda self: %d" % val)
+        if what == "property":
+            val = draw(strategies.integers())
+            return (property(lambda self, val=val: val,
+                             lambda self, val: None,
+                             lambda self: None),
+                    "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+    code = ["import sys", "class OldBase:pass", "class NewBase(object):pass", "class A(%s):" % baseclass]
+    dct = {}
+    if draw(strategies.booleans()):
+        slots = draw(strategies.lists(attrnames))
+        if not hasdict and draw(strategies.booleans()):
+            slots.append("__dict__")
+        dct["__slots__"] = slots
+        code.append("    __slots__ = %s" % (slots, ))
+    for name in ["a", "b", "c"]:
+        if not draw(strategies.booleans()):
+            continue
+        dct[name], codeval = class_attr()
+        code.append("    %s = %s" % (name, codeval))
+    class OldBase: pass
+    class NewBase(object): pass
+    evaldct = {'OldBase': OldBase, 'NewBase': NewBase}
+    if baseclass == 'OldBase':
+        metaclass = type(OldBase)
+    else:
+        metaclass = type
+    cls = metaclass("A", eval(baseclass+',', globals(), evaldct), dct)
+    inst = cls(*initargs)
+    code.append("    pass")
+    code.append("a = A(*%s)" % (initargs, ))
+    for attr in draw(strategies.lists(attrnames, min_size=1)):
+        op = draw(strategies.sampled_from(["read", "read", "read",
+                      "write", "writemeth", "writeclass", "writebase",
+                      "del", "delclass"]))
+        if op == "read":
+            try:
+                res = getattr(inst, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s')" % (attr, ))
+            else:
+                if callable(res):
+                    code.append("assert a.%s() == %s" % (attr, res()))
+                else:
+                    code.append("assert a.%s == %s" % (attr, res))
+        elif op == "write":
+            val = draw(strategies.integers())
+            try:
+                setattr(inst, attr, val)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s=%s')" % (attr, val))
+            else:
+                code.append("a.%s = %s" % (attr, val))
+        elif op == "writemeth":
+            val = draw(strategies.integers())
+            try:
+                setattr(inst, attr, lambda val=val: val)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s=0')" % (attr, ))
+            else:
+                code.append("a.%s = lambda : %s" % (attr, val))
+        elif op == "writeclass":
+            val, codeval = class_attr()
+            setattr(cls, attr, val)
+            code.append("A.%s = %s" % (attr, codeval))
+        elif op == "writebase":
+            val, codeval = class_attr()
+            setattr(OldBase, attr, val)
+            setattr(NewBase, attr, val)
+            code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))
+        elif op == "del":
+            try:
+                delattr(inst, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'del a.%s')" % (attr, ))
+            else:
+                code.append("del a.%s" % (attr, ))
+        elif op == "delclass":
+            try:
+                delattr(cls, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'del A.%s')" % (attr, ))
+            else:
+                code.append("del A.%s" % (attr, ))
+    return "\n    ".join(code)
+
+
+ at given(make_code())
+#@settings(max_examples=5000)
+def test_random_attrs(code):
+    try:
+        import __pypy__
+    except ImportError:
+        pass
+    else:
+        pytest.skip("makes no sense under pypy!")
+    space = gettestobjspace()
+    print code
+    exec "if 1:\n    " + code
+    space.appexec([], "():\n    " + code)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1064,6 +1064,8 @@
         else:
             assert mc.get_relative_pos() <= 13
         mc.copy_to_raw_memory(oldadr)
+        # log the redirection of the call_assembler_* operation
+        jl.redirect_assembler(oldlooptoken, newlooptoken, target)
 
     def dump(self, text):
         if not self.verbose:
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 2
+JITLOG_VERSION = 3
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -244,6 +244,7 @@
     ('COMMON_PREFIX',),
     ('ABORT_TRACE',),
     ('SOURCE_CODE',),
+    ('REDIRECT_ASSEMBLER',),
 ]
 
 start = 0x11
@@ -309,6 +310,17 @@
     content = ''.join(list)
     jitlog_write_marked(content, len(content))
 
+def redirect_assembler(oldtoken, newtoken, target):
+    if not jitlog_enabled():
+        return
+    descr_nmr = compute_unique_id(oldtoken)
+    new_descr_nmr = compute_unique_id(newtoken)
+    list = [MARK_REDIRECT_ASSEMBLER, encode_le_addr(descr_nmr),
+            encode_le_addr(new_descr_nmr), encode_le_addr(target)]
+    content = ''.join(list)
+    jitlog_write_marked(content, len(content))
+
+
 class JitLogger(object):
     def __init__(self, cpu=None):
         self.cpu = cpu
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -6,6 +6,15 @@
 from rpython.jit.backend.model import AbstractCPU
 from rpython.jit.metainterp.history import ConstInt, ConstPtr
 from rpython.rlib.rjitlog import rjitlog as jl
+from rpython.jit.metainterp.history import AbstractDescr
+from rpython.rlib.objectmodel import compute_unique_id
+
+class FakeCallAssemblerLoopToken(AbstractDescr):
+    def __init__(self, target):
+        self._ll_function_addr = target
+
+    def repr_of_descr(self):
+        return 'looptoken'
 
 class FakeLog(object):
     def __init__(self):
@@ -109,3 +118,31 @@
         with py.test.raises(AssertionError):
             jl.commonprefix(None,None)
 
+    def test_redirect_assembler(self, tmpdir):
+        looptoken = FakeCallAssemblerLoopToken(0x0)
+        newlooptoken = FakeCallAssemblerLoopToken(0x1234)
+        #
+        logger = jl.JitLogger()
+        file = tmpdir.join('binary_file')
+        file.ensure()
+        fd = file.open('wb')
+        jl.jitlog_init(fd.fileno())
+        logger.start_new_trace(self.make_metainterp_sd(), jd_name='jdname')
+        log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
+        op = ResOperation(rop.CALL_ASSEMBLER_I, [], descr=looptoken)
+        log_trace.write([], [op])
+        jl.redirect_assembler(looptoken, newlooptoken, 0x1234)
+        #the next line will close 'fd', instead of logger.finish()
+        fd.close()
+        binary = file.read()
+        opnum = jl.encode_le_16bit(rop.CALL_ASSEMBLER_I)
+        id_looptoken = compute_unique_id(looptoken)
+        new_id_looptoken = compute_unique_id(newlooptoken)
+        end = jl.MARK_RESOP_DESCR + opnum + jl.encode_str('i0,looptoken') + \
+              jl.encode_le_addr(id_looptoken) + jl.encode_str('') + \
+              jl.MARK_REDIRECT_ASSEMBLER + \
+              jl.encode_le_addr(id_looptoken) + \
+              jl.encode_le_addr(new_id_looptoken) + \
+              jl.encode_le_addr(newlooptoken._ll_function_addr)
+        assert binary.endswith(end)
+        
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1099,12 +1099,14 @@
         AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
         """
         result = lltype.malloc(_c.socketpair_t, 2, flavor='raw')
-        res = _c.socketpair(family, type, proto, result)
-        if res < 0:
-            raise last_error()
-        fd0 = rffi.cast(lltype.Signed, result[0])
-        fd1 = rffi.cast(lltype.Signed, result[1])
-        lltype.free(result, flavor='raw')
+        try:
+            res = _c.socketpair(family, type, proto, result)
+            if res < 0:
+                raise last_error()
+            fd0 = rffi.cast(lltype.Signed, result[0])
+            fd1 = rffi.cast(lltype.Signed, result[1])
+        finally:
+            lltype.free(result, flavor='raw')
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -237,8 +237,10 @@
                              " directly to a VOIDP argument")
     _oops._annspecialcase_ = 'specialize:memo'
 
+    nb_args = len(args)
     unrolling_arg_tps = unrolling_iterable(enumerate(args))
     def wrapper(*args):
+        assert len(args) == nb_args
         real_args = ()
         # XXX 'to_free' leaks if an allocation fails with MemoryError
         # and was not the first in this function
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -49,6 +49,7 @@
         eci = ExternalCompilationInfo(includes=['stuff.h'],
                                       include_dirs=[udir])
         z = llexternal('X', [Signed], Signed, compilation_info=eci)
+        py.test.raises(AssertionError, z, 8, 9)
 
         def f():
             return z(8)

From pypy.commits at gmail.com  Sat Aug 27 12:31:23 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 Aug 2016 09:31:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: Small cleanup
Message-ID: <57c1c05b.262ec20a.3aef8.4722@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86607:05aa9435b133
Date: 2016-08-27 17:30 +0100
http://bitbucket.org/pypy/pypy/changeset/05aa9435b133/

Log:	Small cleanup

diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -2,6 +2,12 @@
 import sys
 from pypy.tool.pytest.objspace import gettestobjspace
 try:
+    import __pypy__
+except ImportError:
+    pass
+else:
+    pytest.skip("makes no sense under pypy!")
+try:
     from hypothesis import given, strategies, settings
 except ImportError:
     pytest.skip("requires hypothesis")
@@ -119,16 +125,9 @@
     return "\n    ".join(code)
 
 
- at given(make_code())
+ at given(code=make_code())
 #@settings(max_examples=5000)
-def test_random_attrs(code):
-    try:
-        import __pypy__
-    except ImportError:
-        pass
-    else:
-        pytest.skip("makes no sense under pypy!")
-    space = gettestobjspace()
+def test_random_attrs(code, space):
     print code
     exec "if 1:\n    " + code
     space.appexec([], "():\n    " + code)

From pypy.commits at gmail.com  Sat Aug 27 13:01:59 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 Aug 2016 10:01:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Extract class_attr as a separate
 strategy
Message-ID: <57c1c787.0117c20a.c7a53.5dbf@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86608:de272690e3c2
Date: 2016-08-27 18:01 +0100
http://bitbucket.org/pypy/pypy/changeset/de272690e3c2/

Log:	Extract class_attr as a separate strategy

diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -24,26 +24,25 @@
 attrnames = strategies.sampled_from(["a", "b", "c"])
 
 @strategies.composite
+def class_attr(draw):
+    what = draw(strategies.sampled_from(["value", "method", "property"]))
+    if what == "value":
+        val = draw(strategies.integers())
+        return val, str(val)
+    if what == "method":
+        val = draw(strategies.integers())
+        return (lambda self, val=val: val,
+                "lambda self: %d" % val)
+    if what == "property":
+        val = draw(strategies.integers())
+        return (property(lambda self, val=val: val,
+                            lambda self, val: None,
+                            lambda self: None),
+                "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+ at strategies.composite
 def make_code(draw):
-    # now here we can do this kind of thing:
     baseclass, initargs, hasdict = draw(base_initargs)
-    # and with arbitrary strategies
-
-    def class_attr():
-        what = draw(strategies.sampled_from(["value", "method", "property"]))
-        if what == "value":
-            val = draw(strategies.integers())
-            return val, str(val)
-        if what == "method":
-            val = draw(strategies.integers())
-            return (lambda self, val=val: val,
-                    "lambda self: %d" % val)
-        if what == "property":
-            val = draw(strategies.integers())
-            return (property(lambda self, val=val: val,
-                             lambda self, val: None,
-                             lambda self: None),
-                    "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
 
     code = ["import sys", "class OldBase:pass", "class NewBase(object):pass", "class A(%s):" % baseclass]
     dct = {}
@@ -56,7 +55,7 @@
     for name in ["a", "b", "c"]:
         if not draw(strategies.booleans()):
             continue
-        dct[name], codeval = class_attr()
+        dct[name], codeval = draw(class_attr())
         code.append("    %s = %s" % (name, codeval))
     class OldBase: pass
     class NewBase(object): pass
@@ -100,11 +99,11 @@
             else:
                 code.append("a.%s = lambda : %s" % (attr, val))
         elif op == "writeclass":
-            val, codeval = class_attr()
+            val, codeval = draw(class_attr())
             setattr(cls, attr, val)
             code.append("A.%s = %s" % (attr, codeval))
         elif op == "writebase":
-            val, codeval = class_attr()
+            val, codeval = draw(class_attr())
             setattr(OldBase, attr, val)
             setattr(NewBase, attr, val)
             code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))

From pypy.commits at gmail.com  Sat Aug 27 13:06:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 10:06:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: in-progress
Message-ID: <57c1c8ad.8628c20a.5d8b9.5167@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86609:4414cc2fc2f5
Date: 2016-08-27 17:27 +0200
http://bitbucket.org/pypy/pypy/changeset/4414cc2fc2f5/

Log:	in-progress

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -228,7 +228,7 @@
 #     CPython + 7 = default_magic  -- used by PyPy (incompatible!)
 #
 from pypy.interpreter.pycode import default_magic
-MARSHAL_VERSION_FOR_PYC = 2
+MARSHAL_VERSION_FOR_PYC = 3
 
 def get_pyc_magic(space):
     return default_magic
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -4,24 +4,29 @@
 from rpython.rlib import rstackovf
 from pypy.objspace.std.marshal_impl import marshal, get_unmarshallers
 
+#
+# Write Python objects to files and read them back.  This is primarily
+# intended for writing and reading compiled Python code, even though
+# dicts, lists, sets and frozensets, not commonly seen in code
+# objects, are supported.  Version 3 of this protocol properly
+# supports circular links and sharing.  The previous version is called
+# "2", like in Python 2.7, although it is not always compatible
+# between CPython 2.7 and CPython 3.x.
+#
+# XXX: before py3k, there was logic to do efficiently dump()/load() on
+# a file object.  The corresponding logic is gone from CPython 3.x, so
+# I don't feel bad about killing it here too.
+#
 
-Py_MARSHAL_VERSION = 2
+Py_MARSHAL_VERSION = 3
+
 
 @unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION))
 def dump(space, w_data, w_f, w_version):
     """Write the 'data' object into the open file 'f'."""
-    # XXX: before py3k, we special-cased W_File to use a more performant
-    # FileWriter class. Should we do the same for py3k? Look also at
-    # DirectStreamWriter
-    writer = FileWriter(space, w_f)
-    try:
-        # note: bound methods are currently not supported,
-        # so we have to pass the instance in, instead.
-        ##m = Marshaller(space, writer.write, space.int_w(w_version))
-        m = Marshaller(space, writer, space.int_w(w_version))
-        m.dump_w_obj(w_data)
-    finally:
-        writer.finished()
+    # same implementation as CPython 3.x.
+    w_string = dumps(space, w_data, w_version)
+    space.call_method(w_f, 'write', w_string)
 
 @unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION))
 def dumps(space, w_data, w_version):
@@ -33,9 +38,6 @@
 
 def load(space, w_f):
     """Read one value from the file 'f' and return it."""
-    # XXX: before py3k, we special-cased W_File to use a more performant
-    # FileWriter class. Should we do the same for py3k? Look also at
-    # DirectStreamReader
     reader = FileReader(space, w_f)
     try:
         u = Unmarshaller(space, reader)
@@ -68,22 +70,6 @@
     def write(self, data):
         raise NotImplementedError("Purely abstract method")
 
-class FileWriter(AbstractReaderWriter):
-    def __init__(self, space, w_f):
-        AbstractReaderWriter.__init__(self, space)
-        try:
-            self.func = space.getattr(w_f, space.wrap('write'))
-            # XXX how to check if it is callable?
-        except OperationError as e:
-            if not e.match(space, space.w_AttributeError):
-                raise
-            raise oefmt(space.w_TypeError,
-                        "marshal.dump() 2nd arg must be file-like object")
-
-    def write(self, data):
-        space = self.space
-        space.call_function(self.func, space.newbytes(data))
-
 
 class FileReader(AbstractReaderWriter):
     def __init__(self, space, w_f):
@@ -111,33 +97,6 @@
         return ret
 
 
-class StreamReaderWriter(AbstractReaderWriter):
-    def __init__(self, space, file):
-        AbstractReaderWriter.__init__(self, space)
-        self.file = file
-        file.lock()
-
-    def finished(self):
-        self.file.unlock()
-
-class DirectStreamWriter(StreamReaderWriter):
-    """
-    XXX: this class is unused right now. Look at the comment in dump()
-    """
-    def write(self, data):
-        self.file.do_direct_write(data)
-
-class DirectStreamReader(StreamReaderWriter):
-    """
-    XXX: this class is unused right now. Look at the comment in dump()
-    """
-    def read(self, n):
-        data = self.file.direct_read(n)
-        if len(data) < n:
-            self.raise_eof()
-        return data
-
-
 class _Base(object):
     def raise_exc(self, msg):
         space = self.space
@@ -354,7 +313,6 @@
     def __init__(self, space, reader):
         self.space = space
         self.reader = reader
-        self.stringtable_w = []
 
     def get(self, n):
         assert n >= 0
diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py
--- a/pypy/module/marshal/test/test_marshalimpl.py
+++ b/pypy/module/marshal/test/test_marshalimpl.py
@@ -6,20 +6,6 @@
 class AppTestMarshalMore:
     spaceconfig = dict(usemodules=('array',))
 
-    def test_unmarshal_int64(self):
-        # test that we can unmarshal 64-bit ints on 32-bit platforms
-        # (of course we only test that if we're running on such a
-        # platform :-)
-        import marshal
-        z = marshal.loads(b'I\x00\xe4\x0bT\x02\x00\x00\x00')
-        assert z == 10000000000
-        z = marshal.loads(b'I\x00\x1c\xf4\xab\xfd\xff\xff\xff')
-        assert z == -10000000000
-        z = marshal.loads(b'I\x88\x87\x86\x85\x84\x83\x82\x01')
-        assert z == 108793946209421192
-        z = marshal.loads(b'I\xd8\xd8\xd9\xda\xdb\xdc\xcd\xfe')
-        assert z == -0x0132232425262728
-
     def test_marshal_bufferlike_object(self):
         import marshal, array
         s = marshal.dumps(array.array('b', b'asd'))
@@ -33,10 +19,6 @@
     def test_unmarshal_evil_long(self):
         import marshal
         raises(ValueError, marshal.loads, b'l\x02\x00\x00\x00\x00\x00\x00\x00')
-        z = marshal.loads(b'I\x00\xe4\x0bT\x02\x00\x00\x00')
-        assert z == 10000000000
-        z = marshal.loads(b'I\x00\x1c\xf4\xab\xfd\xff\xff\xff')
-        assert z == -10000000000
 
     def test_marshal_code_object(self):
         def foo(a, b):
@@ -49,6 +31,14 @@
             if attr_name.startswith("co_"):
                 assert getattr(code2, attr_name) == getattr(foo.__code__, attr_name)
 
+    def test_shared_string(self):
+        x = "hello, "
+        x += "world"
+        s = marshal.dumps((x, x))
+        assert s.count(x) == 1
+        y = marshal.loads(s)
+        assert y == (x, x)
+
 
 class AppTestMarshalSmallLong(AppTestMarshalMore):
     spaceconfig = dict(usemodules=('array',),
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -29,14 +29,14 @@
 TYPE_STOPITER  = 'S'
 TYPE_ELLIPSIS  = '.'
 TYPE_INT       = 'i'
-TYPE_INT64     = 'I'
 TYPE_FLOAT     = 'f'
 TYPE_BINARY_FLOAT = 'g'
 TYPE_COMPLEX   = 'x'
 TYPE_BINARY_COMPLEX = 'y'
 TYPE_LONG      = 'l'
-TYPE_STRING    = 's'
-TYPE_STRINGREF = 'R'
+TYPE_STRING    = 's'     # a *byte* string, not unicode
+TYPE_INTERNED  = 't'
+TYPE_REF       = 'r'
 TYPE_TUPLE     = '('
 TYPE_LIST      = '['
 TYPE_DICT      = '{'
@@ -45,6 +45,13 @@
 TYPE_UNKNOWN   = '?'
 TYPE_SET       = '<'
 TYPE_FROZENSET = '>'
+FLAG_REF       = 0x80    # bit added to mean "add obj to index"
+
+TYPE_ASCII                = 'a'   # never generated so far
+TYPE_ASCII_INTERNED       = 'A'   # never generated so far
+TYPE_SMALL_TUPLE          = ')'
+TYPE_SHORT_ASCII          = 'z'   # never generated so far
+TYPE_SHORT_ASCII_INTERNED = 'Z'   # never generated so far
 
 
 _marshallers = []
@@ -75,7 +82,8 @@
         s = space.readbuf_w(w_obj)
     except OperationError as e:
         if e.match(space, space.w_TypeError):
-            raise oefmt(space.w_ValueError, "unmarshallable object")
+            raise oefmt(space.w_ValueError, "cannot marshal '%T' object",
+                        w_obj)
         raise
     m.atom_str(TYPE_STRING, s.as_str())
 
@@ -108,7 +116,7 @@
 @marshaller(W_TypeObject)
 def marshal_stopiter(space, w_type, m):
     if not space.is_w(w_type, space.w_StopIteration):
-        raise oefmt(space.w_ValueError, "unmarshallable object")
+        raise oefmt(space.w_ValueError, "cannot marshal type object")
     m.atom(TYPE_STOPITER)
 
 @unmarshaller(TYPE_STOPITER)
@@ -127,37 +135,26 @@
 
 @marshaller(W_IntObject)
 def marshal_int(space, w_int, m):
-    if LONG_BIT == 32:
+    y = w_int.intval >> 31
+    if y and y != -1:
+        _marshal_bigint(space, space.bigint_w(w_int), m)
+    else:
         m.atom_int(TYPE_INT, w_int.intval)
-    else:
-        y = w_int.intval >> 31
-        if y and y != -1:
-            m.atom_int64(TYPE_INT64, w_int.intval)
-        else:
-            m.atom_int(TYPE_INT, w_int.intval)
 
 @unmarshaller(TYPE_INT)
 def unmarshal_int(space, u, tc):
     return space.newint(u.get_int())
 
- at unmarshaller(TYPE_INT64)
-def unmarshal_int64(space, u, tc):
-    lo = u.get_int()    # get the first 32 bits
-    hi = u.get_int()    # get the next 32 bits
-    if LONG_BIT >= 64:
-        x = (hi << 32) | (lo & (2**32-1))    # result fits in an int
-    else:
-        x = (r_longlong(hi) << 32) | r_longlong(r_uint(lo))  # get a r_longlong
-    return space.wrap(x)
-
 
 @marshaller(W_AbstractLongObject)
 def marshal_long(space, w_long, m):
+    _marshal_bigint(space, w_long.asbigint(), m)
+
+def _marshal_bigint(space, num, m):
     from rpython.rlib.rarithmetic import r_ulonglong
     m.start(TYPE_LONG)
     SHIFT = 15
     MASK = (1 << SHIFT) - 1
-    num = w_long.asbigint()
     sign = num.sign
     num = num.abs()
     total_length = (num.bit_length() + (SHIFT - 1)) / SHIFT
@@ -252,14 +249,6 @@
 def unmarshal_bytes(space, u, tc):
     return space.newbytes(u.get_str())
 
- at unmarshaller(TYPE_STRINGREF)
-def unmarshal_stringref(space, u, tc):
-    idx = u.get_int()
-    try:
-        return u.stringtable_w[idx]
-    except IndexError:
-        raise oefmt(space.w_ValueError, "bad marshal data")
-
 
 @marshaller(W_AbstractTupleObject)
 def marshal_tuple(space, w_tuple, m):

From pypy.commits at gmail.com  Sat Aug 27 13:06:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 10:06:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: in-progress
Message-ID: <57c1c8af.0205c20a.f2090.4b28@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86610:fec7aa29e819
Date: 2016-08-27 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/fec7aa29e819/

Log:	in-progress

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -837,13 +837,13 @@
             self.interned_strings.set(u, w_s1)
         return w_s1
 
-    def is_interned_str(self, s):
+    def get_interned_str(self, s):
         """Assumes an identifier (utf-8 encoded str)"""
         # interface for marshal_impl
         if not we_are_translated():
             assert type(s) is str
         u = s.decode('utf-8')
-        return self.interned_strings.get(u) is not None
+        return self.interned_strings.get(u)   # may be None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -228,7 +228,7 @@
 #     CPython + 7 = default_magic  -- used by PyPy (incompatible!)
 #
 from pypy.interpreter.pycode import default_magic
-MARSHAL_VERSION_FOR_PYC = 3
+MARSHAL_VERSION_FOR_PYC = 4
 
 def get_pyc_magic(space):
     return default_magic
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -11,14 +11,14 @@
 # objects, are supported.  Version 3 of this protocol properly
 # supports circular links and sharing.  The previous version is called
 # "2", like in Python 2.7, although it is not always compatible
-# between CPython 2.7 and CPython 3.x.
+# between CPython 2.7 and CPython 3.4.
 #
 # XXX: before py3k, there was logic to do efficiently dump()/load() on
 # a file object.  The corresponding logic is gone from CPython 3.x, so
 # I don't feel bad about killing it here too.
 #
 
-Py_MARSHAL_VERSION = 3
+Py_MARSHAL_VERSION = 4
 
 
 @unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION))
@@ -127,7 +127,15 @@
         ## self.put = putfunc
         self.writer = writer
         self.version = version
-        self.stringtable = {}
+        self.all_refs = {}
+        # all_refs = {w_obj: index} for all w_obj that are of a
+        # "reasonably sharable" type.  CPython checks the refcount of
+        # any object to know if it is sharable, independently of its
+        # type.  We can't do that.  We could do a two-pass marshaller.
+        # For now we simply add to this list all objects that marshal to
+        # more than a few fixed-sized bytes, minus ones like code
+        # objects that never appear more than once except in complete
+        # corner cases.
 
     ## currently we cannot use a put that is a bound method
     ## from outside. Same holds for get.
@@ -207,10 +215,13 @@
             rstackovf.check_stack_overflow()
             self._overflow()
 
-    def put_tuple_w(self, typecode, lst_w):
+    def put_tuple_w(self, typecode, lst_w, single_byte_size=False):
         self.start(typecode)
         lng = len(lst_w)
-        self.put_int(lng)
+        if single_byte_size:
+            self.put(chr(lng))
+        else:
+            self.put_int(lng)
         idx = 0
         while idx < lng:
             w_obj = lst_w[idx]
@@ -301,18 +312,35 @@
 
 
 def invalid_typecode(space, u, tc):
-    u.raise_exc("bad marshal data (unknown type code)")
+    u.raise_exc("bad marshal data (unknown type code %d)" % (ord(tc),))
 
 
+def _make_unmarshall_and_save_ref(func):
+    def unmarshall_save_ref(space, u, tc):
+        index = len(u.refs_w)
+        u.refs_w.append(None)
+        w_obj = func(space, u, tc)
+        u.refs_w[index] = w_obj
+        return w_obj
+    return unmarshall_save_ref
 
-class Unmarshaller(_Base):
+def _make_unmarshaller_dispatch():
     _dispatch = [invalid_typecode] * 256
     for tc, func in get_unmarshallers():
         _dispatch[ord(tc)] = func
+    for tc, func in get_unmarshallers():
+        if tc < '\x80' and _dispatch[ord(tc) + 0x80] is invalid_typecode:
+            _dispatch[ord(tc) + 0x80] = _make_unmarshall_and_save_ref(func)
+    return _dispatch
+
+
+class Unmarshaller(_Base):
+    _dispatch = _make_unmarshaller_dispatch()
 
     def __init__(self, space, reader):
         self.space = space
         self.reader = reader
+        self.refs_w = []
 
     def get(self, n):
         assert n >= 0
@@ -322,6 +350,10 @@
         # the [0] is used to convince the annotator to return a char
         return self.get(1)[0]
 
+    def save_ref(self, typecode, w_obj):
+        if typecode >= '\x80':
+            self.refs_w.append(w_obj)
+
     def atom_str(self, typecode):
         self.start(typecode)
         lng = self.get_lng()
@@ -392,8 +424,11 @@
             self._overflow()
 
     # inlined version to save a recursion level
-    def get_tuple_w(self):
-        lng = self.get_lng()
+    def get_tuple_w(self, single_byte_size=False):
+        if single_byte_size:
+            lng = ord(self.get1())
+        else:
+            lng = self.get_lng()
         res_w = [None] * lng
         idx = 0
         space = self.space
@@ -409,9 +444,6 @@
             raise oefmt(space.w_TypeError, "NULL object in marshal data")
         return res_w
 
-    def get_list_w(self):
-        return self.get_tuple_w()[:]
-
     def _overflow(self):
         self.raise_exc('object too deeply nested to unmarshal')
 
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -197,7 +197,7 @@
     def test_bad_typecode(self):
         import marshal
         exc = raises(ValueError, marshal.loads, bytes([1]))
-        assert str(exc.value) == "bad marshal data (unknown type code)"
+        assert str(exc.value).startswith("bad marshal data (unknown type code")
 
     def test_bad_data(self):
         # If you have sufficiently little memory, the line at the end of the
diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py
--- a/pypy/module/marshal/test/test_marshalimpl.py
+++ b/pypy/module/marshal/test/test_marshalimpl.py
@@ -34,10 +34,21 @@
     def test_shared_string(self):
         x = "hello, "
         x += "world"
-        s = marshal.dumps((x, x))
-        assert s.count(x) == 1
-        y = marshal.loads(s)
-        assert y == (x, x)
+        xl = 256
+        xl **= 100
+        for version in [2, 3]:
+            s = marshal.dumps((x, x), version)
+            assert s.count(x) == 2 if version < 3 else 1
+            y = marshal.loads(s)
+            assert y == (x, x)
+            #
+            s = marshal.dumps((xl, xl), version)
+            if version < 3:
+                assert 200 < len(s) < 250
+            else:
+                assert 100 < len(s) < 125
+            yl = marshal.loads(s)
+            assert yl == (xl, xl)
 
 
 class AppTestMarshalSmallLong(AppTestMarshalMore):
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -2,6 +2,7 @@
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rstruct import ieee
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib import objectmodel
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.special import Ellipsis
@@ -46,12 +47,14 @@
 TYPE_SET       = '<'
 TYPE_FROZENSET = '>'
 FLAG_REF       = 0x80    # bit added to mean "add obj to index"
+FLAG_DONE      = '\x00'
 
-TYPE_ASCII                = 'a'   # never generated so far
-TYPE_ASCII_INTERNED       = 'A'   # never generated so far
+# the following typecodes have been added in version 4.
+TYPE_ASCII                = 'a'   # never generated so far by pypy
+TYPE_ASCII_INTERNED       = 'A'   # never generated so far by pypy
 TYPE_SMALL_TUPLE          = ')'
-TYPE_SHORT_ASCII          = 'z'   # never generated so far
-TYPE_SHORT_ASCII_INTERNED = 'Z'   # never generated so far
+TYPE_SHORT_ASCII          = 'z'   # never generated so far by pypy
+TYPE_SHORT_ASCII_INTERNED = 'Z'   # never generated so far by pypy
 
 
 _marshallers = []
@@ -63,12 +66,33 @@
         return f
     return _decorator
 
-def unmarshaller(tc):
+def unmarshaller(tc, save_ref=False):
     def _decorator(f):
+        assert tc < '\x80'
         _unmarshallers.append((tc, f))
+        if save_ref:
+            tcref = chr(ord(tc) + 0x80)
+            _unmarshallers.append((tcref, f))
         return f
     return _decorator
 
+def write_ref(typecode, w_obj, m):
+    if m.version < 3:
+        return typecode     # not writing object references
+    try:
+        index = m.all_refs[w_obj]
+    except KeyError:
+        # we don't support long indices
+        index = len(m.all_refs)
+        if index >= 0x7fffffff:
+            return typecode
+        m.all_refs[w_obj] = index
+        return chr(ord(typecode) + FLAG_REF)
+    else:
+        # write the reference index to the stream
+        m.atom_int(TYPE_REF, index)
+        return FLAG_DONE
+
 def marshal(space, w_obj, m):
     # _marshallers_unroll is defined at the end of the file
     for type, func in _marshallers_unroll:
@@ -82,10 +106,11 @@
         s = space.readbuf_w(w_obj)
     except OperationError as e:
         if e.match(space, space.w_TypeError):
-            raise oefmt(space.w_ValueError, "cannot marshal '%T' object",
-                        w_obj)
+            raise oefmt(space.w_ValueError, "unmarshallable object")
         raise
-    m.atom_str(TYPE_STRING, s.as_str())
+    typecode = write_ref(TYPE_STRING, w_obj, m)
+    if typecode != FLAG_DONE:
+        m.atom_str(typecode, s.as_str())
 
 def get_unmarshallers():
     return _unmarshallers
@@ -116,7 +141,7 @@
 @marshaller(W_TypeObject)
 def marshal_stopiter(space, w_type, m):
     if not space.is_w(w_type, space.w_StopIteration):
-        raise oefmt(space.w_ValueError, "cannot marshal type object")
+        raise oefmt(space.w_ValueError, "unmarshallable object")
     m.atom(TYPE_STOPITER)
 
 @unmarshaller(TYPE_STOPITER)
@@ -137,7 +162,7 @@
 def marshal_int(space, w_int, m):
     y = w_int.intval >> 31
     if y and y != -1:
-        _marshal_bigint(space, space.bigint_w(w_int), m)
+        marshal_long(space, w_int, m)
     else:
         m.atom_int(TYPE_INT, w_int.intval)
 
@@ -148,13 +173,14 @@
 
 @marshaller(W_AbstractLongObject)
 def marshal_long(space, w_long, m):
-    _marshal_bigint(space, w_long.asbigint(), m)
-
-def _marshal_bigint(space, num, m):
     from rpython.rlib.rarithmetic import r_ulonglong
-    m.start(TYPE_LONG)
+    typecode = write_ref(TYPE_LONG, w_long, m)
+    if typecode == FLAG_DONE:
+        return
+    m.start(typecode)
     SHIFT = 15
     MASK = (1 << SHIFT) - 1
+    num = space.bigint_w(w_long)
     sign = num.sign
     num = num.abs()
     total_length = (num.bit_length() + (SHIFT - 1)) / SHIFT
@@ -242,8 +268,10 @@
 
 @marshaller(W_BytesObject)
 def marshal_bytes(space, w_str, m):
-    s = w_str.unwrap(space)
-    m.atom_str(TYPE_STRING, s)
+    typecode = write_ref(TYPE_STRING, w_str, m)
+    if typecode != FLAG_DONE:
+        s = space.bytes_w(w_str)
+        m.atom_str(typecode, s)
 
 @unmarshaller(TYPE_STRING)
 def unmarshal_bytes(space, u, tc):
@@ -253,40 +281,62 @@
 @marshaller(W_AbstractTupleObject)
 def marshal_tuple(space, w_tuple, m):
     items = w_tuple.tolist()
-    m.put_tuple_w(TYPE_TUPLE, items)
+    if m.version >= 4 and len(items) < 256:
+        typecode = TYPE_SMALL_TUPLE
+        single_byte_size = True
+    else:
+        typecode = TYPE_TUPLE
+        single_byte_size = False
+    typecode = write_ref(typecode, w_tuple, m)
+    if typecode != FLAG_DONE:
+        m.put_tuple_w(typecode, items, single_byte_size=single_byte_size)
 
 @unmarshaller(TYPE_TUPLE)
 def unmarshal_tuple(space, u, tc):
     items_w = u.get_tuple_w()
     return space.newtuple(items_w)
 
+ at unmarshaller(TYPE_SMALL_TUPLE)
+def unmarshal_tuple(space, u, tc):
+    items_w = u.get_tuple_w(single_byte_size=True)
+    return space.newtuple(items_w)
+
 
 @marshaller(W_ListObject)
 def marshal_list(space, w_list, m):
-    items = w_list.getitems()[:]
-    m.put_tuple_w(TYPE_LIST, items)
+    typecode = write_ref(TYPE_LIST, w_list, m)
+    if typecode != FLAG_DONE:
+        items = w_list.getitems()[:]
+        m.put_tuple_w(typecode, items)
 
- at unmarshaller(TYPE_LIST)
+ at unmarshaller(TYPE_LIST, save_ref=True)
 def unmarshal_list(space, u, tc):
-    items_w = u.get_list_w()
-    return space.newlist(items_w)
+    w_obj = space.newlist([])
+    u.save_ref(tc, w_obj)
+    for w_item in u.get_tuple_w():
+        w_obj.append(w_item)
+    return w_obj
 
 
 @marshaller(W_DictMultiObject)
 def marshal_dict(space, w_dict, m):
-    m.start(TYPE_DICT)
+    typecode = write_ref(TYPE_DICT, w_dict, m)
+    if typecode == FLAG_DONE:
+        return
+    m.start(typecode)
     for w_tuple in w_dict.items():
         w_key, w_value = space.fixedview(w_tuple, 2)
         m.put_w_obj(w_key)
         m.put_w_obj(w_value)
     m.atom(TYPE_NULL)
 
- at unmarshaller(TYPE_DICT)
+ at unmarshaller(TYPE_DICT, save_ref=True)
 def unmarshal_dict(space, u, tc):
     # since primitive lists are not optimized and we don't know
     # the dict size in advance, use the dict's setitem instead
     # of building a list of tuples.
     w_dic = space.newdict()
+    u.save_ref(tc, w_dic)
     while 1:
         w_key = u.get_w_obj(allow_null=True)
         if w_key is None:
@@ -308,6 +358,7 @@
 
 @marshaller(PyCode)
 def marshal_pycode(space, w_pycode, m):
+    # (no attempt at using write_ref here, there is little point imho)
     m.start(TYPE_CODE)
     # see pypy.interpreter.pycode for the layout
     x = space.interp_w(PyCode, w_pycode)
@@ -353,8 +404,10 @@
     lng = u.atom_lng(tc)
     return [unmarshal_str(u) for i in range(lng)]
 
- at unmarshaller(TYPE_CODE)
+ at unmarshaller(TYPE_CODE, save_ref=True)
 def unmarshal_pycode(space, u, tc):
+    w_codeobj = objectmodel.instantiate(PyCode)
+    u.save_ref(tc, w_codeobj)
     argcount    = u.get_int()
     kwonlyargcount = u.get_int()
     nlocals     = u.get_int()
@@ -372,50 +425,85 @@
     name        = unmarshal_str(u)
     firstlineno = u.get_int()
     lnotab      = unmarshal_str(u)
-    return PyCode(space, argcount, kwonlyargcount, nlocals, stacksize, flags,
+    PyCode.__init__(w_codeobj,
+                  space, argcount, kwonlyargcount, nlocals, stacksize, flags,
                   code, consts_w[:], names, varnames, filename,
                   name, firstlineno, lnotab, freevars, cellvars)
+    return w_codeobj
 
 
 @marshaller(W_UnicodeObject)
 def marshal_unicode(space, w_unicode, m):
     s = unicodehelper.encode_utf8(space, space.unicode_w(w_unicode),
                                   allow_surrogates=True)
-    m.atom_str(TYPE_UNICODE, s)
+    if m.version >= 3:
+        w_interned = space.get_interned_str(s)
+    else:
+        w_interned = None
+    if w_interned is not None:
+        w_unicode = w_interned    # use the interned W_UnicodeObject
+        typecode = TYPE_INTERNED  #   as a key for u.all_refs
+    else:
+        typecode = TYPE_UNICODE
+    typecode = write_ref(typecode, w_unicode, m)
+    if typecode != FLAG_DONE:
+        m.atom_str(typecode, s)
 
 @unmarshaller(TYPE_UNICODE)
 def unmarshal_unicode(space, u, tc):
-    return space.wrap(unicodehelper.decode_utf8(space, u.get_str(),
-                                                allow_surrogates=True))
+    uc = unicodehelper.decode_utf8(space, u.get_str(), allow_surrogates=True)
+    return space.newunicode(uc)
+
+ at unmarshaller(TYPE_INTERNED)
+def unmarshal_bytes(space, u, tc):
+    return space.new_interned_str(u.get_str())
+
 
 @marshaller(W_SetObject)
 def marshal_set(space, w_set, m):
-    lis_w = space.fixedview(w_set)
-    m.put_tuple_w(TYPE_SET, lis_w)
+    typecode = write_ref(TYPE_SET, w_set, m)
+    if typecode != FLAG_DONE:
+        lis_w = space.fixedview(w_set)
+        m.put_tuple_w(typecode, lis_w)
 
- at unmarshaller(TYPE_SET)
+ at unmarshaller(TYPE_SET, save_ref=True)
 def unmarshal_set(space, u, tc):
-    return unmarshal_set_frozenset(space, u, tc)
+    w_set = space.call_function(space.w_set)
+    u.save_ref(tc, w_set)
+    _unmarshal_set_frozenset(space, u, w_set)
+    return w_set
 
 
 @marshaller(W_FrozensetObject)
 def marshal_frozenset(space, w_frozenset, m):
-    lis_w = space.fixedview(w_frozenset)
-    m.put_tuple_w(TYPE_FROZENSET, lis_w)
+    typecode = write_ref(TYPE_FROZENSET, w_frozenset, m)
+    if typecode != FLAG_DONE:
+        lis_w = space.fixedview(w_frozenset)
+        m.put_tuple_w(typecode, lis_w)
 
-def unmarshal_set_frozenset(space, u, tc):
+def _unmarshal_set_frozenset(space, u, w_set):
     lng = u.get_lng()
-    w_set = space.call_function(space.w_set)
     for i in xrange(lng):
         w_obj = u.get_w_obj()
         space.call_method(w_set, "add", w_obj)
-    if tc == TYPE_FROZENSET:
-        w_set = space.call_function(space.w_frozenset, w_set)
-    return w_set
 
 @unmarshaller(TYPE_FROZENSET)
 def unmarshal_frozenset(space, u, tc):
-    return unmarshal_set_frozenset(space, u, tc)
+    w_set = space.call_function(space.w_set)
+    _unmarshal_set_frozenset(space, u, w_set)
+    return space.call_function(space.w_frozenset, w_set)
+
+
+ at unmarshaller(TYPE_REF)
+def unmarshal_ref(space, u, tc):
+    index = u.get_lng()
+    if 0 <= index < len(u.refs_w):
+        w_obj = u.refs_w[index]
+    else:
+        w_obj = None
+    if w_obj is None:
+        raise oefmt(space.w_ValueError, "bad marshal data (invalid reference)")
+    return w_obj
 
 
 _marshallers_unroll = unrolling_iterable(_marshallers)

From pypy.commits at gmail.com  Sat Aug 27 13:15:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 10:15:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: Hopefully a correct fix for
 marshal.dumps() with objects that are
Message-ID: <57c1ca95.8628c20a.5d8b9.53cb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86611:6810d750b914
Date: 2016-08-27 19:14 +0200
http://bitbucket.org/pypy/pypy/changeset/6810d750b914/

Log:	Hopefully a correct fix for marshal.dumps() with objects that are
	subclasses of one of the built-in class. The previous logic was only
	applied on the outermost object, not on any object referenced from
	there.

diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -225,15 +225,6 @@
 
     def dump_w_obj(self, w_obj):
         space = self.space
-        if space.type(w_obj).is_heaptype():
-            try:
-                buf = space.readbuf_w(w_obj)
-            except OperationError as e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                self.raise_exc("unmarshallable object")
-            else:
-                w_obj = space.newbuffer(buf)
         try:
             self.put_w_obj(w_obj)
         except rstackovf.StackOverflow:
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -186,6 +186,8 @@
             assert str(exc.value) == 'unmarshallable object'
             exc = raises(ValueError, marshal.dumps, subtype())
             assert str(exc.value) == 'unmarshallable object'
+            exc = raises(ValueError, marshal.dumps, (subtype(),))
+            assert str(exc.value) == 'unmarshallable object'
 
     def test_valid_subtypes(self):
         import marshal
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -65,10 +65,13 @@
 
 def marshal(space, w_obj, m):
     # _marshallers_unroll is defined at the end of the file
-    for type, func in _marshallers_unroll:
-        if isinstance(w_obj, type):
-            func(space, w_obj, m)
-            return
+    # NOTE that if w_obj is a heap type, like an instance of a
+    # user-defined subclass, then we skip that part completely!
+    if not space.type(w_obj).is_heaptype():
+        for type, func in _marshallers_unroll:
+            if isinstance(w_obj, type):
+                func(space, w_obj, m)
+                return
 
     # any unknown object implementing the buffer protocol is
     # accepted and encoded as a plain string

From pypy.commits at gmail.com  Sat Aug 27 13:16:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 10:16:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c1cae5.a717c20a.2f7c9.5d6c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86613:58fe0ad0ec9d
Date: 2016-08-27 19:15 +0200
http://bitbucket.org/pypy/pypy/changeset/58fe0ad0ec9d/

Log:	hg merge py3k

diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -233,15 +233,6 @@
 
     def dump_w_obj(self, w_obj):
         space = self.space
-        if space.type(w_obj).is_heaptype():
-            try:
-                buf = space.readbuf_w(w_obj)
-            except OperationError as e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                self.raise_exc("unmarshallable object")
-            else:
-                w_obj = space.newbuffer(buf)
         try:
             self.put_w_obj(w_obj)
         except rstackovf.StackOverflow:
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -186,6 +186,8 @@
             assert str(exc.value) == 'unmarshallable object'
             exc = raises(ValueError, marshal.dumps, subtype())
             assert str(exc.value) == 'unmarshallable object'
+            exc = raises(ValueError, marshal.dumps, (subtype(),))
+            assert str(exc.value) == 'unmarshallable object'
 
     def test_valid_subtypes(self):
         import marshal
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -64,10 +64,13 @@
 
 def marshal(space, w_obj, m):
     # _marshallers_unroll is defined at the end of the file
-    for type, func in _marshallers_unroll:
-        if isinstance(w_obj, type):
-            func(space, w_obj, m)
-            return
+    # NOTE that if w_obj is a heap type, like an instance of a
+    # user-defined subclass, then we skip that part completely!
+    if not space.type(w_obj).is_heaptype():
+        for type, func in _marshallers_unroll:
+            if isinstance(w_obj, type):
+                func(space, w_obj, m)
+                return
 
     # any unknown object implementing the buffer protocol is
     # accepted and encoded as a plain string
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -2,6 +2,12 @@
 import sys
 from pypy.tool.pytest.objspace import gettestobjspace
 try:
+    import __pypy__
+except ImportError:
+    pass
+else:
+    pytest.skip("makes no sense under pypy!")
+try:
     from hypothesis import given, strategies, settings
 except ImportError:
     pytest.skip("requires hypothesis")
@@ -18,26 +24,25 @@
 attrnames = strategies.sampled_from(["a", "b", "c"])
 
 @strategies.composite
+def class_attr(draw):
+    what = draw(strategies.sampled_from(["value", "method", "property"]))
+    if what == "value":
+        val = draw(strategies.integers())
+        return val, str(val)
+    if what == "method":
+        val = draw(strategies.integers())
+        return (lambda self, val=val: val,
+                "lambda self: %d" % val)
+    if what == "property":
+        val = draw(strategies.integers())
+        return (property(lambda self, val=val: val,
+                            lambda self, val: None,
+                            lambda self: None),
+                "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+ at strategies.composite
 def make_code(draw):
-    # now here we can do this kind of thing:
     baseclass, initargs, hasdict = draw(base_initargs)
-    # and with arbitrary strategies
-
-    def class_attr():
-        what = draw(strategies.sampled_from(["value", "method", "property"]))
-        if what == "value":
-            val = draw(strategies.integers())
-            return val, str(val)
-        if what == "method":
-            val = draw(strategies.integers())
-            return (lambda self, val=val: val,
-                    "lambda self: %d" % val)
-        if what == "property":
-            val = draw(strategies.integers())
-            return (property(lambda self, val=val: val,
-                             lambda self, val: None,
-                             lambda self: None),
-                    "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
 
     code = ["import sys", "class OldBase:pass", "class NewBase(object):pass", "class A(%s):" % baseclass]
     dct = {}
@@ -50,7 +55,7 @@
     for name in ["a", "b", "c"]:
         if not draw(strategies.booleans()):
             continue
-        dct[name], codeval = class_attr()
+        dct[name], codeval = draw(class_attr())
         code.append("    %s = %s" % (name, codeval))
     class OldBase: pass
     class NewBase(object): pass
@@ -94,11 +99,11 @@
             else:
                 code.append("a.%s = lambda : %s" % (attr, val))
         elif op == "writeclass":
-            val, codeval = class_attr()
+            val, codeval = draw(class_attr())
             setattr(cls, attr, val)
             code.append("A.%s = %s" % (attr, codeval))
         elif op == "writebase":
-            val, codeval = class_attr()
+            val, codeval = draw(class_attr())
             setattr(OldBase, attr, val)
             setattr(NewBase, attr, val)
             code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))
@@ -119,16 +124,9 @@
     return "\n    ".join(code)
 
 
- at given(make_code())
+ at given(code=make_code())
 #@settings(max_examples=5000)
-def test_random_attrs(code):
-    try:
-        import __pypy__
-    except ImportError:
-        pass
-    else:
-        pytest.skip("makes no sense under pypy!")
-    space = gettestobjspace()
+def test_random_attrs(code, space):
     print code
     exec "if 1:\n    " + code
     space.appexec([], "():\n    " + code)

From pypy.commits at gmail.com  Sat Aug 27 13:16:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 10:16:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: test fix
Message-ID: <57c1cae7.8a13c20a.23ed6.4d67@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86614:9eb66e9bbf64
Date: 2016-08-27 19:15 +0200
http://bitbucket.org/pypy/pypy/changeset/9eb66e9bbf64/

Log:	test fix

diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py
--- a/pypy/module/marshal/test/test_marshalimpl.py
+++ b/pypy/module/marshal/test/test_marshalimpl.py
@@ -32,6 +32,7 @@
                 assert getattr(code2, attr_name) == getattr(foo.__code__, attr_name)
 
     def test_shared_string(self):
+        import marshal
         x = "hello, "
         x += "world"
         xl = 256

From pypy.commits at gmail.com  Sat Aug 27 13:16:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 10:16:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c1cae3.c4ebc20a.62fc2.516c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86612:782495d7d9eb
Date: 2016-08-27 19:15 +0200
http://bitbucket.org/pypy/pypy/changeset/782495d7d9eb/

Log:	hg merge default

diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -233,15 +233,6 @@
 
     def dump_w_obj(self, w_obj):
         space = self.space
-        if space.type(w_obj).is_heaptype():
-            try:
-                buf = space.readbuf_w(w_obj)
-            except OperationError as e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                self.raise_exc("unmarshallable object")
-            else:
-                w_obj = space.newbuffer(buf)
         try:
             self.put_w_obj(w_obj)
         except rstackovf.StackOverflow:
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -186,6 +186,8 @@
             assert str(exc.value) == 'unmarshallable object'
             exc = raises(ValueError, marshal.dumps, subtype())
             assert str(exc.value) == 'unmarshallable object'
+            exc = raises(ValueError, marshal.dumps, (subtype(),))
+            assert str(exc.value) == 'unmarshallable object'
 
     def test_valid_subtypes(self):
         import marshal
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -64,10 +64,13 @@
 
 def marshal(space, w_obj, m):
     # _marshallers_unroll is defined at the end of the file
-    for type, func in _marshallers_unroll:
-        if isinstance(w_obj, type):
-            func(space, w_obj, m)
-            return
+    # NOTE that if w_obj is a heap type, like an instance of a
+    # user-defined subclass, then we skip that part completely!
+    if not space.type(w_obj).is_heaptype():
+        for type, func in _marshallers_unroll:
+            if isinstance(w_obj, type):
+                func(space, w_obj, m)
+                return
 
     # any unknown object implementing the buffer protocol is
     # accepted and encoded as a plain string
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -2,6 +2,12 @@
 import sys
 from pypy.tool.pytest.objspace import gettestobjspace
 try:
+    import __pypy__
+except ImportError:
+    pass
+else:
+    pytest.skip("makes no sense under pypy!")
+try:
     from hypothesis import given, strategies, settings
 except ImportError:
     pytest.skip("requires hypothesis")
@@ -18,26 +24,25 @@
 attrnames = strategies.sampled_from(["a", "b", "c"])
 
 @strategies.composite
+def class_attr(draw):
+    what = draw(strategies.sampled_from(["value", "method", "property"]))
+    if what == "value":
+        val = draw(strategies.integers())
+        return val, str(val)
+    if what == "method":
+        val = draw(strategies.integers())
+        return (lambda self, val=val: val,
+                "lambda self: %d" % val)
+    if what == "property":
+        val = draw(strategies.integers())
+        return (property(lambda self, val=val: val,
+                            lambda self, val: None,
+                            lambda self: None),
+                "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+ at strategies.composite
 def make_code(draw):
-    # now here we can do this kind of thing:
     baseclass, initargs, hasdict = draw(base_initargs)
-    # and with arbitrary strategies
-
-    def class_attr():
-        what = draw(strategies.sampled_from(["value", "method", "property"]))
-        if what == "value":
-            val = draw(strategies.integers())
-            return val, str(val)
-        if what == "method":
-            val = draw(strategies.integers())
-            return (lambda self, val=val: val,
-                    "lambda self: %d" % val)
-        if what == "property":
-            val = draw(strategies.integers())
-            return (property(lambda self, val=val: val,
-                             lambda self, val: None,
-                             lambda self: None),
-                    "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
 
     code = ["import sys", "class OldBase:pass", "class NewBase(object):pass", "class A(%s):" % baseclass]
     dct = {}
@@ -50,7 +55,7 @@
     for name in ["a", "b", "c"]:
         if not draw(strategies.booleans()):
             continue
-        dct[name], codeval = class_attr()
+        dct[name], codeval = draw(class_attr())
         code.append("    %s = %s" % (name, codeval))
     class OldBase: pass
     class NewBase(object): pass
@@ -94,11 +99,11 @@
             else:
                 code.append("a.%s = lambda : %s" % (attr, val))
         elif op == "writeclass":
-            val, codeval = class_attr()
+            val, codeval = draw(class_attr())
             setattr(cls, attr, val)
             code.append("A.%s = %s" % (attr, codeval))
         elif op == "writebase":
-            val, codeval = class_attr()
+            val, codeval = draw(class_attr())
             setattr(OldBase, attr, val)
             setattr(NewBase, attr, val)
             code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))
@@ -119,16 +124,9 @@
     return "\n    ".join(code)
 
 
- at given(make_code())
+ at given(code=make_code())
 #@settings(max_examples=5000)
-def test_random_attrs(code):
-    try:
-        import __pypy__
-    except ImportError:
-        pass
-    else:
-        pytest.skip("makes no sense under pypy!")
-    space = gettestobjspace()
+def test_random_attrs(code, space):
     print code
     exec "if 1:\n    " + code
     space.appexec([], "():\n    " + code)

From pypy.commits at gmail.com  Sat Aug 27 13:16:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 10:16:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: hg merge py3.5
Message-ID: <57c1cae9.11051c0a.563fc.90bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86615:e75acfc7d246
Date: 2016-08-27 19:15 +0200
http://bitbucket.org/pypy/pypy/changeset/e75acfc7d246/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 2720 lines

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -554,6 +554,9 @@
     def putwin(self, filep):
         # filestar = ffi.new("FILE *", filep)
         return _check_ERR(lib.putwin(self._win, filep), "putwin")
+        # XXX CPython 3.5 says: We have to simulate this by writing to
+        # a temporary FILE*, then reading back, then writing to the
+        # argument stream.
 
     def redrawln(self, beg, num):
         return _check_ERR(lib.wredrawln(self._win, beg, num), "redrawln")
@@ -704,6 +707,7 @@
 
 
 def getwin(filep):
+    # XXX CPython 3.5: there's logic to use a temp file instead
     return Window(_check_NULL(lib.getwin(filep)))
 
 
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,219 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. 
+
+XXXXX MORE ???
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Support for memoryview attributes (format, itemsize, ...) which also
+    adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,157 +1,8 @@
-=========================
-What's new in PyPy2.7 5.3+
-=========================
+==========================
+What's new in PyPy2.7 5.4+
+==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 4176c6f63109
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
 
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
-
-
-.. branch: const-fold-we-are-jitted
-
-Reduce the size of the generated C code by constant-folding ``we_are_jitted``
-in non-jitcode.
-
-.. branch: memoryview-attributes
-
-Support for memoryview attributes (format, itemsize, ...).
-Extends the cpyext emulation layer.
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -0,0 +1,165 @@
+=========================
+What's new in PyPy2.7 5.4
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
+
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -4,6 +4,7 @@
     OperationError, oefmt, wrap_oserror, wrap_oserror2)
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import rposix
 from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC, O_EXCL
 import sys, os, stat, errno
 from pypy.module._io.interp_iobase import W_RawIOBase, convert_size
@@ -29,6 +30,7 @@
 
 O_BINARY = getattr(os, "O_BINARY", 0)
 O_APPEND = getattr(os, "O_APPEND", 0)
+_open_inhcache = rposix.SetNonInheritableCache()
 
 def _bad_mode(space):
     raise oefmt(space.w_ValueError,
@@ -139,6 +141,7 @@
 
     @unwrap_spec(mode=str, closefd=int)
     def descr_init(self, space, w_name, mode='r', closefd=True, w_opener=None):
+        self._close(space)
         if space.isinstance_w(w_name, space.w_float):
             raise oefmt(space.w_TypeError,
                         "integer argument expected, got float")
@@ -153,6 +156,8 @@
                 raise oefmt(space.w_ValueError, "negative file descriptor")
 
         self.readable, self.writable, self.created, self.appending, flags = decode_mode(space, mode)
+        if rposix.O_CLOEXEC is not None:
+            flags |= rposix.O_CLOEXEC
 
         fd_is_own = False
         try:
@@ -171,8 +176,7 @@
                     raise oefmt(space.w_ValueError,
                                 "Cannot use closefd=False with file name")
 
-                from pypy.module.posix.interp_posix import (
-                    dispatch_filename, rposix)
+                from pypy.module.posix.interp_posix import dispatch_filename
                 try:
                     self.fd = dispatch_filename(rposix.open)(
                         space, w_name, flags, 0666)
@@ -181,6 +185,11 @@
                                         exception_name='w_IOError')
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    try:
+                        _open_inhcache.set_non_inheritable(self.fd)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
             else:
                 w_fd = space.call_function(w_opener, w_name, space.wrap(flags))
                 try:
@@ -192,6 +201,11 @@
                                 "expected integer from opener")
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    try:
+                        rposix.set_inheritable(self.fd, False)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
 
             self._dircheck(space, w_name)
             space.setattr(self, space.wrap("name"), w_name)
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -246,6 +246,33 @@
             assert f.mode == 'xb'
         raises(FileExistsError, _io.FileIO, filename, 'x')
 
+    def test_non_inheritable(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        assert posix.get_inheritable(f.fileno()) == False
+        f.close()
+
+    def test_FileIO_fd_does_not_change_inheritable(self):
+        import _io, posix
+        fd1, fd2 = posix.pipe()
+        posix.set_inheritable(fd1, True)
+        posix.set_inheritable(fd2, False)
+        f1 = _io.FileIO(fd1, 'r')
+        f2 = _io.FileIO(fd2, 'w')
+        assert posix.get_inheritable(fd1) == True
+        assert posix.get_inheritable(fd2) == False
+        f1.close()
+        f2.close()
+
+    def test_close_upon_reinit(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        fd1 = f.fileno()
+        f.__init__(self.tmpfile, 'w')
+        fd2 = f.fileno()
+        if fd1 != fd2:
+            raises(OSError, posix.close, fd1)
+
 
 def test_flush_at_exit():
     from pypy import conftest
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.c b/pypy/module/_posixsubprocess/_posixsubprocess.c
--- a/pypy/module/_posixsubprocess/_posixsubprocess.c
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.c
@@ -106,6 +106,30 @@
 }
 
 
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable);   /* rposix.py */
+
+static int
+make_inheritable(long *py_fds_to_keep, ssize_t num_fds_to_keep,
+                 int errpipe_write)
+{
+    long i;
+
+    for (i = 0; i < num_fds_to_keep; ++i) {
+        long fd = py_fds_to_keep[i];
+        if (fd == errpipe_write) {
+            /* errpipe_write is part of py_fds_to_keep. It must be closed at
+               exec(), but kept open in the child process until exec() is
+               called. */
+            continue;
+        }
+        if (rpy_set_inheritable((int)fd, 1) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+
 /* Close all file descriptors in the range start_fd inclusive to
  * end_fd exclusive except for those in py_fds_to_keep.  If the
  * range defined by [start_fd, end_fd) is large this will take a
@@ -329,6 +353,9 @@
     /* Buffer large enough to hold a hex integer.  We can't malloc. */
     char hex_errno[sizeof(saved_errno)*2+1];
 
+    if (make_inheritable(py_fds_to_keep, num_fds_to_keep, errpipe_write) < 0)
+        goto error;
+
     /* Close parent's pipe ends. */
     if (p2cwrite != -1) {
         POSIX_CALL(close(p2cwrite));
@@ -352,26 +379,25 @@
        dup2() removes the CLOEXEC flag but we must do it ourselves if dup2()
        would be a no-op (issue #10806). */
     if (p2cread == 0) {
-        int old = fcntl(p2cread, F_GETFD);
-        if (old != -1)
-            fcntl(p2cread, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (p2cread != -1) {
+        if (rpy_set_inheritable(p2cread, 1) < 0)
+            goto error;
+    }
+    else if (p2cread != -1)
         POSIX_CALL(dup2(p2cread, 0));  /* stdin */
+
+    if (c2pwrite == 1) {
+        if (rpy_set_inheritable(c2pwrite, 1) < 0)
+            goto error;
     }
-    if (c2pwrite == 1) {
-        int old = fcntl(c2pwrite, F_GETFD);
-        if (old != -1)
-            fcntl(c2pwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (c2pwrite != -1) {
+    else if (c2pwrite != -1)
         POSIX_CALL(dup2(c2pwrite, 1));  /* stdout */
+
+    if (errwrite == 2) {
+        if (rpy_set_inheritable(errwrite, 1) < 0)
+            goto error;
     }
-    if (errwrite == 2) {
-        int old = fcntl(errwrite, F_GETFD);
-        if (old != -1)
-            fcntl(errwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (errwrite != -1) {
+    else if (errwrite != -1)
         POSIX_CALL(dup2(errwrite, 2));  /* stderr */
-    }
 
     /* Close pipe fds.  Make sure we don't close the same fd more than */
     /* once, or standard fds. */
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.h b/pypy/module/_posixsubprocess/_posixsubprocess.h
--- a/pypy/module/_posixsubprocess/_posixsubprocess.h
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.h
@@ -1,3 +1,4 @@
+#include <unistd.h>    /* for ssize_t */
 #include "src/precommondefs.h"
 
 RPY_EXTERN void
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -5,6 +5,7 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib import rposix
 
 from pypy.interpreter.error import (
     OperationError, exception_from_saved_errno, oefmt, wrap_oserror)
@@ -36,6 +37,7 @@
     compile_extra.append("-DHAVE_SETSID")
 
 eci = eci.merge(
+    rposix.eci_inheritable,
     ExternalCompilationInfo(
         compile_extra=compile_extra))
 
diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py
--- a/pypy/module/_posixsubprocess/test/test_subprocess.py
+++ b/pypy/module/_posixsubprocess/test/test_subprocess.py
@@ -75,3 +75,18 @@
         n = 1
         raises(OverflowError, _posixsubprocess.fork_exec,
                1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17)
+
+    def test_pass_fds_make_inheritable(self):
+        import subprocess, posix
+
+        fd1, fd2 = posix.pipe()
+        assert posix.get_inheritable(fd1) is False
+        assert posix.get_inheritable(fd2) is False
+
+        subprocess.check_call(['/usr/bin/env', 'python', '-c',
+                               'import os;os.write(%d,b"K")' % fd2],
+                              close_fds=True, pass_fds=[fd2])
+        res = posix.read(fd1, 1)
+        assert res == b"K"
+        posix.close(fd1)
+        posix.close(fd2)
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -143,24 +143,11 @@
 @unwrap_spec(fd=int)
 def dup(space, fd):
     try:
-        newfd = rsocket.dup(fd)
+        newfd = rsocket.dup(fd, inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
     return space.wrap(newfd)
 
- at unwrap_spec(fd=int, family=int, type=int, proto=int)
-def fromfd(space, fd, family, type, proto=0):
-    """fromfd(fd, family, type[, proto]) -> socket object
-
-    Create a socket object from the given file descriptor.
-    The remaining arguments are the same as for socket().
-    """
-    try:
-        sock = rsocket.fromfd(fd, family, type, proto)
-    except SocketError as e:
-        raise converted_error(space, e)
-    return space.wrap(W_Socket(space, sock))
-
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
                       type  =rsocket.SOCK_STREAM,
@@ -173,7 +160,8 @@
     AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
     """
     try:
-        sock1, sock2 = rsocket.socketpair(family, type, proto)
+        sock1, sock2 = rsocket.socketpair(family, type, proto,
+                                          inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
     return space.newtuple([
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -177,7 +177,7 @@
                 sock = RSocket(family, type, proto,
                                fd=space.c_filedescriptor_w(w_fileno))
             else:
-                sock = RSocket(family, type, proto)
+                sock = RSocket(family, type, proto, inheritable=False)
             W_Socket.__init__(self, space, sock)
         except SocketError as e:
             raise converted_error(space, e)
@@ -228,7 +228,7 @@
         For IP sockets, the address info is a pair (hostaddr, port).
         """
         try:
-            fd, addr = self.sock.accept()
+            fd, addr = self.sock.accept(inheritable=False)
             return space.newtuple([space.wrap(fd),
                                    addr_as_object(addr, fd, space)])
         except SocketError as e:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -546,11 +546,15 @@
         s.ioctl(_socket.SIO_KEEPALIVE_VALS, (1, 100, 100))
 
     def test_dup(self):
-        import _socket as socket
+        import _socket as socket, posix
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.bind(('localhost', 0))
         fd = socket.dup(s.fileno())
         assert s.fileno() != fd
+        assert posix.get_inheritable(s.fileno()) is False
+        assert posix.get_inheritable(fd) is False
+        posix.close(fd)
+        s.close()
 
     def test_dup_error(self):
         import _socket
@@ -652,6 +656,26 @@
         assert len(w) == 1, [str(warning) for warning in w]
         assert r in str(w[0])
 
+    def test_invalid_fd(self):
+        import _socket
+        raises(ValueError, _socket.socket, fileno=-1)
+
+    def test_socket_non_inheritable(self):
+        import _socket, posix
+        s1 = _socket.socket()
+        assert posix.get_inheritable(s1.fileno()) is False
+        s1.close()
+
+    def test_socketpair_non_inheritable(self):
+        import _socket, posix
+        if not hasattr(_socket, 'socketpair'):
+            skip("no socketpair")
+        s1, s2 = _socket.socketpair()
+        assert posix.get_inheritable(s1.fileno()) is False
+        assert posix.get_inheritable(s2.fileno()) is False
+        s1.close()
+        s2.close()
+
 
 class AppTestNetlink:
     def setup_class(cls):
@@ -830,6 +854,16 @@
         assert cli.family == socket.AF_INET
 
 
+    def test_accept_non_inheritable(self):
+        import _socket, posix
+        cli = _socket.socket()
+        cli.connect(self.serv.getsockname())
+        fileno, addr = self.serv._accept()
+        assert posix.get_inheritable(fileno) is False
+        posix.close(fileno)
+        cli.close()
+
+
 class AppTestErrno:
     spaceconfig = {'usemodules': ['_socket']}
 
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -356,9 +356,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -458,7 +464,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -610,7 +616,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -154,6 +154,7 @@
 
     def test_readValues(self):
         from winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -167,7 +168,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,7 +119,7 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 Py_CLEANUP_SUPPORTED
@@ -975,13 +975,15 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
     global py_fatalerror
     py_fatalerror = rffi.llexternal('%s_FatalError' % prefix,
                                     [CONST_STRING], lltype.Void,
                                     compilation_info=eci)
-    add_fork_hook('child', reinit_tls)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
+    add_fork_hook('child', _reinit_tls)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.4.1-alpha0"
+#define PYPY_VERSION_NUM  0x05040100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -1,3 +1,6 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
 #include <Python.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -10,7 +13,7 @@
 /* Structure defines a 1-dimensional strided array */
 typedef struct{
     int* arr;
-    long length;
+    Py_ssize_t length;
 } MyArray;
 
 /* initialize the array with integers 0...length */
@@ -61,13 +64,13 @@
 static int
 PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
 {
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
     // init may have already been called
     if (self->arr.arr != NULL) {
         deallocate_MyArray(&self->arr);
     }
 
-    int length = 0;
-    static char *kwlist[] = {"length", NULL};
     if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
         return -1;
 
@@ -103,16 +106,19 @@
 static int
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
 
-  PyMyArray* self = (PyMyArray*)obj;
   view->obj = (PyObject*)self;
   view->buf = (void*)self->arr.arr;
   view->len = self->arr.length * sizeof(int);
@@ -218,7 +224,6 @@
 #ifdef __GNUC__
 extern __attribute__((visibility("default")))
 #else
-extern __declspec(dllexport)
 #endif
 
 PyMODINIT_FUNC
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -77,4 +77,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -18,7 +18,8 @@
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER)
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -386,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -721,8 +724,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -32,7 +32,7 @@
 
         f = open(self.tmp + "b", "w+")
 
-        fcntl.fcntl(f, 1, 0)
+        original = fcntl.fcntl(f, 1, 0)
         fcntl.fcntl(f, 1)
         fcntl.fcntl(F(int(f.fileno())), 1)
         raises(TypeError, fcntl.fcntl, "foo")
@@ -46,9 +46,16 @@
         raises(ValueError, fcntl.fcntl, -1, 1, 0)
         raises(ValueError, fcntl.fcntl, F(-1), 1, 0)
         raises(ValueError, fcntl.fcntl, F(int(-1)), 1, 0)
-        assert fcntl.fcntl(f, 1, 0) == 0
+        assert fcntl.fcntl(f, 1, 0) == original
         assert fcntl.fcntl(f, 2, "foo") == b"foo"
-        assert fcntl.fcntl(f, 2, memoryview(b"foo")) == b"foo"
+        assert fcntl.fcntl(f, 2, b"foo") == b"foo"
+
+        # This is supposed to work I think, but CPython 3.5 refuses it
+        # for reasons I don't understand:
+        #     >>> _testcapi.getargs_s_hash(memoryview(b"foo"))
+        #     TypeError: must be read-only bytes-like object, not memoryview
+        #
+        # assert fcntl.fcntl(f, 2, memoryview(b"foo")) == b"foo"
 
         try:
             os.O_LARGEFILE
@@ -202,7 +209,7 @@
         raises(TypeError, fcntl.ioctl, "foo")
         raises(TypeError, fcntl.ioctl, 0, "foo")
         #raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, float(0))
-        raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, 1, "foo")
+        raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, 1, "foo", "bar")
 
         child_pid, mfd = pty.fork()
         if child_pid == 0:
@@ -229,13 +236,13 @@
             assert res == 0
             assert buf.tostring() == expected
 
-            exc = raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, memoryview(b'abc'), False)
-            assert str(exc.value) == "ioctl requires a file or file descriptor, an integer and optionally an integer or buffer argument"
+            raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, (), False)
 
             res = fcntl.ioctl(mfd, TIOCGPGRP, buf, False)
             assert res == expected
 
-            raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, "\x00\x00", True)
+            # xxx this fails on CPython 3.5, that's a minor bug
+            #raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, "\x00\x00", True)
 
             res = fcntl.ioctl(mfd, TIOCGPGRP, "\x00\x00\x00\x00")
             assert res == expected
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -200,15 +200,6 @@
 
     def dump_w_obj(self, w_obj):
         space = self.space
-        if space.type(w_obj).is_heaptype():
-            try:
-                buf = space.readbuf_w(w_obj)
-            except OperationError as e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                self.raise_exc("unmarshallable object")
-            else:
-                w_obj = space.newbuffer(buf)
         try:
             self.put_w_obj(w_obj)
         except rstackovf.StackOverflow:
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -186,6 +186,8 @@
             assert str(exc.value) == 'unmarshallable object'
             exc = raises(ValueError, marshal.dumps, subtype())
             assert str(exc.value) == 'unmarshallable object'
+            exc = raises(ValueError, marshal.dumps, (subtype(),))
+            assert str(exc.value) == 'unmarshallable object'
 
     def test_valid_subtypes(self):
         import marshal
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -78,6 +78,8 @@
         'get_terminal_size': 'interp_posix.get_terminal_size',
 
         'scandir': 'interp_scandir.scandir',
+        'get_inheritable': 'interp_posix.get_inheritable',
+        'set_inheritable': 'interp_posix.set_inheritable',
     }
 
     if hasattr(os, 'chown'):
@@ -195,6 +197,9 @@
     interpleveldefs['_have_functions'] = (
         'space.newlist([space.wrap(x) for x in interp_posix.have_functions])')
 
+    if rposix.HAVE_PIPE2:
+        interpleveldefs['pipe2'] = 'interp_posix.pipe2'
+
     def startup(self, space):
         from pypy.module.posix import interp_posix
         from pypy.module.imp import importing
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -211,6 +211,8 @@
             space.w_NotImplementedError,
             "%s: %s unavailable on this platform", funcname, arg)
 
+_open_inhcache = rposix.SetNonInheritableCache()
+
 @unwrap_spec(flags=c_int, mode=c_int, dir_fd=DirFD(rposix.HAVE_OPENAT))
 def open(space, w_path, flags, mode=0777,
          __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
@@ -222,12 +224,15 @@
   and path should be relative; path will then be relative to that directory.
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
+    if rposix.O_CLOEXEC is not None:
+        flags |= rposix.O_CLOEXEC
     try:
         if rposix.HAVE_OPENAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             fd = rposix.openat(path, flags, mode, dir_fd)
         else:
             fd = dispatch_filename(rposix.open)(space, w_path, flags, mode)
+        _open_inhcache.set_non_inheritable(fd)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     return space.wrap(fd)
@@ -538,17 +543,17 @@
     """Create a copy of the file descriptor.  Return the new file
 descriptor."""
     try:
-        newfd = os.dup(fd)
+        newfd = rposix.dup(fd, inheritable=False)
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
         return space.wrap(newfd)
 
- at unwrap_spec(old_fd=c_int, new_fd=c_int)
-def dup2(space, old_fd, new_fd):
+ at unwrap_spec(old_fd=c_int, new_fd=c_int, inheritable=int)
+def dup2(space, old_fd, new_fd, inheritable=1):
     """Duplicate a file descriptor."""
     try:
-        os.dup2(old_fd, new_fd)
+        rposix.dup2(old_fd, new_fd, inheritable)
     except OSError as e:
         raise wrap_oserror(space, e)
 
@@ -891,15 +896,38 @@
             result_w[i] = space.fsdecode(w_bytes)
     return space.newlist(result_w)
 
+ at unwrap_spec(fd=c_int)
+def get_inheritable(space, fd):
+    try:
+        return space.wrap(rposix.get_inheritable(fd))
+    except OSError as e:
+        raise wrap_oserror(space, e)
+
+ at unwrap_spec(fd=c_int, inheritable=int)
+def set_inheritable(space, fd, inheritable):
+    try:
+        rposix.set_inheritable(fd, inheritable)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+
+_pipe_inhcache = rposix.SetNonInheritableCache()
+
 def pipe(space):
     "Create a pipe.  Returns (read_end, write_end)."
     try:
-        fd1, fd2 = os.pipe()
+        fd1, fd2 = rposix.pipe(rposix.O_CLOEXEC or 0)
+        _pipe_inhcache.set_non_inheritable(fd1)
+        _pipe_inhcache.set_non_inheritable(fd2)
     except OSError as e:
         raise wrap_oserror(space, e)
-    # XXX later, use rposix.pipe2() if available!
-    rposix.set_inheritable(fd1, False)
-    rposix.set_inheritable(fd2, False)
+    return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
+
+ at unwrap_spec(flags=c_int)
+def pipe2(space, flags):
+    try:
+        fd1, fd2 = rposix.pipe2(flags)
+    except OSError as e:
+        raise wrap_oserror(space, e)
     return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),
@@ -1238,6 +1266,8 @@
     "Open a pseudo-terminal, returning open fd's for both master and slave end."
     try:
         master_fd, slave_fd = os.openpty()
+        rposix.set_inheritable(master_fd, False)
+        rposix.set_inheritable(slave_fd, False)
     except OSError as e:
         raise wrap_oserror(space, e)
     return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)])
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -106,6 +106,7 @@
         posix = self.posix
         fd = posix.open(path, posix.O_RDONLY, 0o777)
         fd2 = posix.dup(fd)
+        assert posix.get_inheritable(fd2) == False
         assert not posix.isatty(fd2)
         s = posix.read(fd, 1)
         assert s == b't'
@@ -398,6 +399,16 @@
             os.write(slave_fd, b'x\n')
             data = os.read(master_fd, 100)
             assert data.startswith(b'x')
+            os.close(master_fd)
+            os.close(slave_fd)
+
+        def test_openpty_non_inheritable(self):
+            os = self.posix
+            master_fd, slave_fd = os.openpty()
+            assert os.get_inheritable(master_fd) == False
+            assert os.get_inheritable(slave_fd) == False
+            os.close(master_fd)
+            os.close(slave_fd)
 
     if hasattr(__import__(os.name), "forkpty"):
         def test_forkpty(self):
@@ -1077,6 +1088,52 @@
         x = f.read(1)
         assert x == 'e'
 
+    def test_pipe_inheritable(self):
+        fd1, fd2 = self.posix.pipe()
+        assert self.posix.get_inheritable(fd1) == False
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_pipe2(self):
+        if not hasattr(self.posix, 'pipe2'):
+            skip("no pipe2")
+        fd1, fd2 = self.posix.pipe2(0)
+        assert self.posix.get_inheritable(fd1) == True
+        assert self.posix.get_inheritable(fd2) == True
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_O_CLOEXEC(self):
+        if not hasattr(self.posix, 'pipe2'):
+            skip("no pipe2")
+        if not hasattr(self.posix, 'O_CLOEXEC'):
+            skip("no O_CLOEXEC")
+        fd1, fd2 = self.posix.pipe2(self.posix.O_CLOEXEC)
+        assert self.posix.get_inheritable(fd1) == False
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_dup2_inheritable(self):
+        fd1, fd2 = self.posix.pipe()
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.dup2(fd1, fd2)
+        assert self.posix.get_inheritable(fd2) == True
+        self.posix.dup2(fd1, fd2, False)
+        assert self.posix.get_inheritable(fd2) == False
+        self.posix.dup2(fd1, fd2, True)
+        assert self.posix.get_inheritable(fd2) == True
+        self.posix.close(fd1)
+        self.posix.close(fd2)
+
+    def test_open_inheritable(self):
+        os = self.posix
+        fd = os.open(self.path2 + 'test_open_inheritable',
+                     os.O_RDWR | os.O_CREAT, 0o666)
+        assert os.get_inheritable(fd) == False
+        os.close(fd)
+
     def test_urandom(self):
         os = self.posix
         s = os.urandom(5)
diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py
--- a/pypy/module/select/interp_epoll.py
+++ b/pypy/module/select/interp_epoll.py
@@ -39,7 +39,8 @@
 for symbol in public_symbols:
     setattr(CConfig, symbol, rffi_platform.DefinedConstantInteger(symbol))
 
-for symbol in ["EPOLL_CTL_ADD", "EPOLL_CTL_MOD", "EPOLL_CTL_DEL"]:
+for symbol in ["EPOLL_CTL_ADD", "EPOLL_CTL_MOD", "EPOLL_CTL_DEL",
+               "EPOLL_CLOEXEC"]:
     setattr(CConfig, symbol, rffi_platform.ConstantInteger(symbol))
 
 cconfig = rffi_platform.configure(CConfig)
@@ -52,13 +53,14 @@
 EPOLL_CTL_ADD = cconfig["EPOLL_CTL_ADD"]
 EPOLL_CTL_MOD = cconfig["EPOLL_CTL_MOD"]
 EPOLL_CTL_DEL = cconfig["EPOLL_CTL_DEL"]
+EPOLL_CLOEXEC = cconfig["EPOLL_CLOEXEC"]
 
 DEF_REGISTER_EVENTMASK = (public_symbols["EPOLLIN"] |
                           public_symbols["EPOLLOUT"] |
                           public_symbols["EPOLLPRI"])
 
-epoll_create = rffi.llexternal(
-    "epoll_create", [rffi.INT], rffi.INT, compilation_info=eci,
+epoll_create1 = rffi.llexternal(
+    "epoll_create1", [rffi.INT], rffi.INT, compilation_info=eci,
     save_err=rffi.RFFI_SAVE_ERRNO
 )
 epoll_ctl = rffi.llexternal(
@@ -82,14 +84,12 @@
         self.epfd = epfd
         self.register_finalizer(space)
 
-    @unwrap_spec(sizehint=int)
-    def descr__new__(space, w_subtype, sizehint=-1):
-        if sizehint == -1:
-            sizehint = FD_SETSIZE - 1
-        elif sizehint < 0:
+    @unwrap_spec(sizehint=int, flags=int)
+    def descr__new__(space, w_subtype, sizehint=0, flags=0):
+        if sizehint < 0:     # 'sizehint' is otherwise ignored
             raise oefmt(space.w_ValueError,
                         "sizehint must be greater than zero, got %d", sizehint)
-        epfd = epoll_create(sizehint)
+        epfd = epoll_create1(flags | EPOLL_CLOEXEC)
         if epfd < 0:
             raise exception_from_saved_errno(space, space.w_IOError)
 
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -1,10 +1,11 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.error import exception_from_saved_errno
+from pypy.interpreter.error import exception_from_saved_errno, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.typedef import TypeDef, generic_new_descr, GetSetProperty
 from rpython.rlib._rsocket_rffi import socketclose_no_errno
 from rpython.rlib.rarithmetic import r_uint
+from rpython.rlib import rposix
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -115,6 +116,10 @@
         kqfd = syscall_kqueue()
         if kqfd < 0:
             raise exception_from_saved_errno(space, space.w_IOError)
+        try:
+            rposix.set_inheritable(kqfd, False)
+        except OSError as e:
+            raise wrap_oserror(space, e)
         return space.wrap(W_Kqueue(space, kqfd))
 
     @unwrap_spec(fd=int)
diff --git a/pypy/module/select/test/test_devpoll.py b/pypy/module/select/test/test_devpoll.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/select/test/test_devpoll.py
@@ -0,0 +1,4 @@
+# XXX
+
+# devpoll is not implemented, but if we do, make sure we test for
+# non-inheritable file descriptors
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -209,3 +209,10 @@
         ep = select.epoll()
         ep.close()
         ep.close()
+
+    def test_non_inheritable(self):
+        import select, posix
+
+        ep = select.epoll()
+        assert posix.get_inheritable(ep.fileno()) == False
+        ep.close()
diff --git a/pypy/module/select/test/test_kqueue.py b/pypy/module/select/test/test_kqueue.py
--- a/pypy/module/select/test/test_kqueue.py
+++ b/pypy/module/select/test/test_kqueue.py
@@ -186,3 +186,10 @@
         a.close()
         b.close()
         kq.close()
+
+    def test_non_inheritable(self):
+        import select, posix
+
+        kq = select.kqueue()
+        assert posix.get_inheritable(kq.fileno()) == False
+        kq.close()
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 2, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/module/test_lib_pypy/test_gdbm_extra.py b/pypy/module/test_lib_pypy/test_gdbm_extra.py
--- a/pypy/module/test_lib_pypy/test_gdbm_extra.py
+++ b/pypy/module/test_lib_pypy/test_gdbm_extra.py
@@ -15,3 +15,7 @@
     assert len(g) == 2
     del g['abc']
     assert len(g) == 1
+
+def test_unicode():
+    path = unicode(udir.join('test_gdm_unicode'))
+    g = gdbm.open(path, 'c')  # does not crash
diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import sys
 
 import os
 if os.name != 'posix':
@@ -47,6 +48,9 @@
     # minimal "does not crash" test
     x, y = resource.getrlimit(resource.RLIMIT_CPU)
     resource.setrlimit(resource.RLIMIT_CPU, (x, y))
-    x += 0.2
-    y += 0.3
-    resource.setrlimit(resource.RLIMIT_CPU, (x, y))    # truncated to ints
+    # sometimes, x and y are very large (more than 53 bits).
+    # for these huge values, int(float(x)) > x...
+    xf = x + (0.2 if x >= 0 else -0.2)
+    yf = y + (0.3 if y >= 0 else -0.3)
+    if int(xf) == x and int(yf) == y:
+        resource.setrlimit(resource.RLIMIT_CPU, (x, y))  # truncated to ints
diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -346,9 +346,16 @@
 
     def descr_hash(self, space):
         hashreal = _hash_float(space, self.realval)
-        hashimg = _hash_float(space, self.imagval)
-        combined = intmask(hashreal + HASH_IMAG * hashimg)
-        return space.newint(-2 if combined == -1 else combined)
+        hashimg = _hash_float(space, self.imagval)   # 0 if self.imagval == 0
+        h = intmask(hashreal + HASH_IMAG * hashimg)
+        h -= (h == -1)
+        return space.newint(h)
+
+    def descr_coerce(self, space, w_other):
+        w_other = self._to_complex(space, w_other)
+        if w_other is None:
+            return space.w_NotImplemented
+        return space.newtuple([self, w_other])
 
     def descr_format(self, space, w_format_spec):
         return newformat.run_formatter(space, w_format_spec, "format_complex",
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -399,7 +399,9 @@
     descr_str = func_with_new_name(descr_repr, 'descr_str')
 
     def descr_hash(self, space):
-        return space.wrap(_hash_float(space, self.floatval))
+        h = _hash_float(space, self.floatval)
+        h -= (h == -1)
+        return space.wrap(h)
 
     def descr_format(self, space, w_spec):
         return newformat.run_formatter(space, w_spec, "format_float", self)
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -991,7 +991,8 @@
             if index != INVALID:
                 attr = map.find_map_attr(attrname, index)
                 if attr is not None:
-                    # Note that if map.terminator is a DevolvedDictTerminator,
+                    # Note that if map.terminator is a DevolvedDictTerminator
+                    # or the class provides its own dict, not using mapdict, then:
                     # map.find_map_attr will always return None if index==DICT.
                     _fill_cache(pycode, nameindex, map, version_tag, attr.storageindex)
                     return w_obj._mapdict_read_storage(attr.storageindex)
@@ -1013,6 +1014,12 @@
 
 def LOOKUP_METHOD_mapdict_fill_cache_method(space, pycode, name, nameindex,
                                             w_obj, w_type, w_method):
+    # if the layout has a dict itself, then mapdict is not used for normal
+    # attributes. Then the cache won't be able to spot changes to the dict.
+    # Thus we don't cache. see test_bug_builtin_types_callmethod
+    if w_type.layout.typedef.hasdict:
+        return
+
     if w_method is None or isinstance(w_method, MutableCell):
         # don't cache the MutableCell XXX could be fixed
         return
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -95,10 +95,13 @@
 
 def marshal(space, w_obj, m):
     # _marshallers_unroll is defined at the end of the file
-    for type, func in _marshallers_unroll:
-        if isinstance(w_obj, type):
-            func(space, w_obj, m)
-            return
+    # NOTE that if w_obj is a heap type, like an instance of a
+    # user-defined subclass, then we skip that part completely!
+    if not space.type(w_obj).is_heaptype():
+        for type, func in _marshallers_unroll:
+            if isinstance(w_obj, type):
+                func(space, w_obj, m)
+                return
 
     # any unknown object implementing the buffer protocol is
     # accepted and encoded as a plain string
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -99,6 +99,8 @@
             buf = SubBuffer(self.buf, start * itemsize, size * itemsize)
             return W_MemoryView(buf, self.format, itemsize)
         else:
+            # XXX needs to return a W_MemoryView with a NonContiguousSubBuffer
+            # maybe?  Need to check the cpyext requirements for that
             raise oefmt(space.w_NotImplementedError,
                         "XXX extended slicing")
 
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -671,3 +671,6 @@
         assert sign(z2.real) == -1
         assert sign(z2.real) == -1
         
+    def test_hash_minus_one(self):
+        assert hash(-1.0 + 0j) == -2
+        assert (-1.0 + 0j).__hash__() == -2
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -475,6 +475,10 @@
         s = '\U0001D7CF\U0001D7CE.4' # &#120783;&#120782;.4
         assert float(s) == 10.4
 
+    def test_hash_minus_one(self):
+        assert hash(-1.0) == -2
+        assert (-1.0).__hash__() == -2
+
 
 class AppTestFloatHex:
     spaceconfig = {
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1238,6 +1238,42 @@
         got = x.a
         assert got == 'd'
 
+    def test_bug_builtin_types_callmethod(self):
+        import sys
+        class D(type(sys)):
+            def mymethod(self):
+                return "mymethod"
+
+        def foobar():
+            return "foobar"
+
+        d = D('d')
+        res1 = d.mymethod()
+        d.mymethod = foobar
+        res2 = d.mymethod()
+        assert res1 == "mymethod"
+        assert res2 == "foobar"
+
+    def test_bug_builtin_types_load_attr(self):
+        import sys
+        class D(type(sys)):
+            def mymethod(self):
+                return "mymethod"
+
+        def foobar():
+            return "foobar"
+
+        d = D('d')
+        m = d.mymethod
+        res1 = m()
+        d.mymethod = foobar
+        m = d.mymethod
+        res2 = m()
+        assert res1 == "mymethod"
+        assert res2 == "foobar"
+
+
+
 class AppTestGlobalCaching(AppTestWithMapDict):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
 
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -0,0 +1,132 @@
+import pytest
+import sys
+from pypy.tool.pytest.objspace import gettestobjspace
+try:
+    import __pypy__
+except ImportError:
+    pass
+else:
+    pytest.skip("makes no sense under pypy!")
+try:
+    from hypothesis import given, strategies, settings
+except ImportError:
+    pytest.skip("requires hypothesis")
+
+base_initargs = strategies.sampled_from([
+    ("object", (), False),
+    ("type(sys)", ("fake", ), True),
+    ("NewBase", (), True),
+    ("OldBase", (), False),
+    ("object, OldBase", (), False),
+    ("type(sys), OldBase", ("fake", ), True),
+    ])
+
+attrnames = strategies.sampled_from(["a", "b", "c"])
+
+ at strategies.composite
+def class_attr(draw):
+    what = draw(strategies.sampled_from(["value", "method", "property"]))
+    if what == "value":
+        val = draw(strategies.integers())
+        return val, str(val)
+    if what == "method":
+        val = draw(strategies.integers())
+        return (lambda self, val=val: val,
+                "lambda self: %d" % val)
+    if what == "property":
+        val = draw(strategies.integers())
+        return (property(lambda self, val=val: val,
+                            lambda self, val: None,
+                            lambda self: None),
+                "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+ at strategies.composite
+def make_code(draw):
+    baseclass, initargs, hasdict = draw(base_initargs)
+
+    code = ["import sys", "class OldBase:pass", "class NewBase(object):pass", "class A(%s):" % baseclass]
+    dct = {}
+    if draw(strategies.booleans()):
+        slots = draw(strategies.lists(attrnames))
+        if not hasdict and draw(strategies.booleans()):
+            slots.append("__dict__")
+        dct["__slots__"] = slots
+        code.append("    __slots__ = %s" % (slots, ))
+    for name in ["a", "b", "c"]:
+        if not draw(strategies.booleans()):
+            continue
+        dct[name], codeval = draw(class_attr())
+        code.append("    %s = %s" % (name, codeval))
+    class OldBase: pass
+    class NewBase(object): pass
+    evaldct = {'OldBase': OldBase, 'NewBase': NewBase}
+    if baseclass == 'OldBase':
+        metaclass = type(OldBase)
+    else:
+        metaclass = type
+    cls = metaclass("A", eval(baseclass+',', globals(), evaldct), dct)
+    inst = cls(*initargs)
+    code.append("    pass")
+    code.append("a = A(*%s)" % (initargs, ))
+    for attr in draw(strategies.lists(attrnames, min_size=1)):
+        op = draw(strategies.sampled_from(["read", "read", "read",
+                      "write", "writemeth", "writeclass", "writebase",
+                      "del", "delclass"]))
+        if op == "read":
+            try:
+                res = getattr(inst, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s')" % (attr, ))
+            else:
+                if callable(res):
+                    code.append("assert a.%s() == %s" % (attr, res()))
+                else:
+                    code.append("assert a.%s == %s" % (attr, res))
+        elif op == "write":
+            val = draw(strategies.integers())
+            try:
+                setattr(inst, attr, val)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s=%s')" % (attr, val))
+            else:
+                code.append("a.%s = %s" % (attr, val))
+        elif op == "writemeth":
+            val = draw(strategies.integers())
+            try:
+                setattr(inst, attr, lambda val=val: val)
+            except AttributeError:
+                code.append("raises(AttributeError, 'a.%s=0')" % (attr, ))
+            else:
+                code.append("a.%s = lambda : %s" % (attr, val))
+        elif op == "writeclass":
+            val, codeval = draw(class_attr())
+            setattr(cls, attr, val)
+            code.append("A.%s = %s" % (attr, codeval))
+        elif op == "writebase":
+            val, codeval = draw(class_attr())
+            setattr(OldBase, attr, val)
+            setattr(NewBase, attr, val)
+            code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))
+        elif op == "del":
+            try:
+                delattr(inst, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'del a.%s')" % (attr, ))
+            else:
+                code.append("del a.%s" % (attr, ))
+        elif op == "delclass":
+            try:
+                delattr(cls, attr)
+            except AttributeError:
+                code.append("raises(AttributeError, 'del A.%s')" % (attr, ))
+            else:
+                code.append("del A.%s" % (attr, ))
+    return "\n    ".join(code)
+
+
+ at given(code=make_code())
+#@settings(max_examples=5000)
+def test_random_attrs(code, space):
+    print code
+    exec "if 1:\n    " + code
+    space.appexec([], "():\n    " + code)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1064,6 +1064,8 @@
         else:
             assert mc.get_relative_pos() <= 13
         mc.copy_to_raw_memory(oldadr)
+        # log the redirection of the call_assembler_* operation
+        jl.redirect_assembler(oldlooptoken, newlooptoken, target)
 
     def dump(self, text):
         if not self.verbose:
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -176,6 +176,7 @@

From pypy.commits at gmail.com  Sat Aug 27 13:19:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 10:19:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: more test fixes
Message-ID: <57c1cbb1.915c1c0a.f8309.8cfb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86616:f46a9864a91a
Date: 2016-08-27 19:19 +0200
http://bitbucket.org/pypy/pypy/changeset/f46a9864a91a/

Log:	more test fixes

diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py
--- a/pypy/module/marshal/test/test_marshalimpl.py
+++ b/pypy/module/marshal/test/test_marshalimpl.py
@@ -39,7 +39,7 @@
         xl **= 100
         for version in [2, 3]:
             s = marshal.dumps((x, x), version)
-            assert s.count(x) == 2 if version < 3 else 1
+            assert s.count(b'hello, world') == 2 if version < 3 else 1
             y = marshal.loads(s)
             assert y == (x, x)
             #
@@ -64,6 +64,7 @@
         # NOTE: marshal is platform independent, running this test must assume
         # that self.seen gets values from the endianess of the marshal module.
         # (which is little endian!)
+        version = 2
         def __init__(self):
             self.seen = []
         def start(self, code):

From pypy.commits at gmail.com  Sat Aug 27 14:41:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 11:41:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: Fixing marshalling of code
 objects as an attempt to be compatible with
Message-ID: <57c1deef.0cce1c0a.b81b5.9dba@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86617:bc23b603172b
Date: 2016-08-27 20:41 +0200
http://bitbucket.org/pypy/pypy/changeset/bc23b603172b/

Log:	Fixing marshalling of code objects as an attempt to be compatible
	with CPython.

diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -48,7 +48,7 @@
         from rpython.config.translationoption import CACHE_DIR
         from pypy.module.marshal import interp_marshal
 
-        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_%s' % (name,))
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib%s' % (name,))
         try:
             if space.config.translating:
                 raise IOError("don't use the cache when translating pypy")
diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py
--- a/pypy/module/marshal/test/test_marshalimpl.py
+++ b/pypy/module/marshal/test/test_marshalimpl.py
@@ -31,6 +31,17 @@
             if attr_name.startswith("co_"):
                 assert getattr(code2, attr_name) == getattr(foo.__code__, attr_name)
 
+    def test_unmarshal_ascii(self):
+        import marshal
+        s = marshal.loads(b"a\x04\x00\x00\x00ab\xc2\x84")
+        assert s == "ab\xc2\x84"
+        s = marshal.loads(b"A\x04\x00\x00\x00ab\xc2\x84")
+        assert s == "ab\xc2\x84"
+        s = marshal.loads(b"z\x04ab\xc2\x84")
+        assert s == "ab\xc2\x84"
+        s = marshal.loads(b"Z\x04ab\xc2\x84")
+        assert s == "ab\xc2\x84"
+
     def test_shared_string(self):
         import marshal
         x = "hello, "
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -281,18 +281,22 @@
     return space.newbytes(u.get_str())
 
 
- at marshaller(W_AbstractTupleObject)
-def marshal_tuple(space, w_tuple, m):
-    items = w_tuple.tolist()
-    if m.version >= 4 and len(items) < 256:
+def _marshal_tuple(space, tuple_w, m):
+    if m.version >= 4 and len(tuple_w) < 256:
         typecode = TYPE_SMALL_TUPLE
         single_byte_size = True
     else:
         typecode = TYPE_TUPLE
         single_byte_size = False
-    typecode = write_ref(typecode, w_tuple, m)
-    if typecode != FLAG_DONE:
-        m.put_tuple_w(typecode, items, single_byte_size=single_byte_size)
+    # -- does it make any sense to try to share tuples, based on the
+    # -- *identity* of the tuple object?  I'd guess not really
+    #typecode = write_ref(typecode, w_tuple, m)
+    #if typecode != FLAG_DONE:
+    m.put_tuple_w(typecode, tuple_w, single_byte_size=single_byte_size)
+
+ at marshaller(W_AbstractTupleObject)
+def marshal_tuple(space, w_tuple, m):
+    _marshal_tuple(space, w_tuple.tolist(), m)
 
 @unmarshaller(TYPE_TUPLE)
 def unmarshal_tuple(space, u, tc):
@@ -353,12 +357,6 @@
     return None
 
 
-def _put_str_list(space, m, strlist):
-    m.atom_int(TYPE_TUPLE, len(strlist))
-    atom_str = m.atom_str
-    for item in strlist:
-        atom_str(TYPE_STRING, item)
-
 @marshaller(PyCode)
 def marshal_pycode(space, w_pycode, m):
     # (no attempt at using write_ref here, there is little point imho)
@@ -371,42 +369,35 @@
     m.put_int(x.co_stacksize)
     m.put_int(x.co_flags)
     m.atom_str(TYPE_STRING, x.co_code)
-    m.put_tuple_w(TYPE_TUPLE, x.co_consts_w)
-    _put_str_list(space, m, [space.str_w(w_name) for w_name in x.co_names_w])
-    _put_str_list(space, m, x.co_varnames)
-    _put_str_list(space, m, x.co_freevars)
-    _put_str_list(space, m, x.co_cellvars)
-    m.atom_str(TYPE_STRING, x.co_filename)
-    m.atom_str(TYPE_STRING, x.co_name)
+    _marshal_tuple(space, x.co_consts_w, m)
+    _marshal_tuple(space, x.co_names_w, m)   # list of w_unicodes
+    co_varnames_w = [space.wrap(s.decode('utf-8')) for s in x.co_varnames]
+    co_freevars_w = [space.wrap(s.decode('utf-8')) for s in x.co_freevars]
+    co_cellvars_w = [space.wrap(s.decode('utf-8')) for s in x.co_cellvars]
+    _marshal_tuple(space, co_varnames_w, m)  # more lists, now of w_unicodes
+    _marshal_tuple(space, co_freevars_w, m)
+    _marshal_tuple(space, co_cellvars_w, m)
+    _marshal_unicode(space, x.co_filename, m)
+    _marshal_unicode(space, x.co_name, m)
     m.put_int(x.co_firstlineno)
     m.atom_str(TYPE_STRING, x.co_lnotab)
 
 # helper for unmarshalling "tuple of string" objects
 # into rpython-level lists of strings.  Only for code objects.
 
-def unmarshal_str(u):
+def _unmarshal_strlist(u):
+    items_w = _unmarshal_tuple_w(u)
+    return [u.space.unicode_w(w_item).encode('utf-8') for w_item in items_w]
+
+def _unmarshal_tuple_w(u):
     w_obj = u.get_w_obj()
     try:
-        return u.space.bytes_w(w_obj)
-    except OperationError as e:
-        if e.match(u.space, u.space.w_TypeError):
-            u.raise_exc('invalid marshal data for code object')
-        else:
-            raise
-
-def unmarshal_str0(u):
-    w_obj = u.get_w_obj()
-    try:
-        return u.space.bytes0_w(w_obj)
+        return u.space.fixedview(w_obj)
     except OperationError as e:
         if e.match(u.space, u.space.w_TypeError):
             u.raise_exc('invalid marshal data for code object')
         raise
 
-def unmarshal_strlist(u, tc):
-    lng = u.atom_lng(tc)
-    return [unmarshal_str(u) for i in range(lng)]
-
 @unmarshaller(TYPE_CODE, save_ref=True)
 def unmarshal_pycode(space, u, tc):
     w_codeobj = objectmodel.instantiate(PyCode)
@@ -416,18 +407,16 @@
     nlocals     = u.get_int()
     stacksize   = u.get_int()
     flags       = u.get_int()
-    code        = unmarshal_str(u)
-    u.start(TYPE_TUPLE)
-    consts_w    = u.get_tuple_w()
-    # copy in order not to merge it with anything else
-    names       = unmarshal_strlist(u, TYPE_TUPLE)
-    varnames    = unmarshal_strlist(u, TYPE_TUPLE)
-    freevars    = unmarshal_strlist(u, TYPE_TUPLE)
-    cellvars    = unmarshal_strlist(u, TYPE_TUPLE)
-    filename    = unmarshal_str0(u)
-    name        = unmarshal_str(u)
+    code        = space.bytes_w(u.get_w_obj())
+    consts_w    = _unmarshal_tuple_w(u)   
+    names       = _unmarshal_strlist(u)
+    varnames    = _unmarshal_strlist(u)
+    freevars    = _unmarshal_strlist(u)
+    cellvars    = _unmarshal_strlist(u)
+    filename    = space.unicode_w(u.get_w_obj()).encode('utf-8')
+    name        = space.unicode_w(u.get_w_obj()).encode('utf-8')
     firstlineno = u.get_int()
-    lnotab      = unmarshal_str(u)
+    lnotab      = space.bytes_w(u.get_w_obj())
     PyCode.__init__(w_codeobj,
                   space, argcount, kwonlyargcount, nlocals, stacksize, flags,
                   code, consts_w[:], names, varnames, filename,
@@ -435,10 +424,7 @@
     return w_codeobj
 
 
- at marshaller(W_UnicodeObject)
-def marshal_unicode(space, w_unicode, m):
-    s = unicodehelper.encode_utf8(space, space.unicode_w(w_unicode),
-                                  allow_surrogates=True)
+def _marshal_unicode(space, s, m, w_unicode=None):
     if m.version >= 3:
         w_interned = space.get_interned_str(s)
     else:
@@ -448,10 +434,17 @@
         typecode = TYPE_INTERNED  #   as a key for u.all_refs
     else:
         typecode = TYPE_UNICODE
-    typecode = write_ref(typecode, w_unicode, m)
+    if w_unicode is not None:
+        typecode = write_ref(typecode, w_unicode, m)
     if typecode != FLAG_DONE:
         m.atom_str(typecode, s)
 
+ at marshaller(W_UnicodeObject)
+def marshal_unicode(space, w_unicode, m):
+    s = unicodehelper.encode_utf8(space, space.unicode_w(w_unicode),
+                                  allow_surrogates=True)
+    _marshal_unicode(space, s, m, w_unicode=w_unicode)
+
 @unmarshaller(TYPE_UNICODE)
 def unmarshal_unicode(space, u, tc):
     uc = unicodehelper.decode_utf8(space, u.get_str(), allow_surrogates=True)
@@ -461,6 +454,19 @@
 def unmarshal_bytes(space, u, tc):
     return space.new_interned_str(u.get_str())
 
+ at unmarshaller(TYPE_ASCII)    # nb. never generated by pypy so far
+def unmarshal_ascii(space, u, tc):
+    _unmarshal_ascii(False, False)
+ at unmarshaller(TYPE_ASCII_INTERNED)
+def unmarshal_ascii(space, u, tc):
+    _unmarshal_ascii(False, True)
+ at unmarshaller(TYPE_SHORT_ASCII)
+def unmarshal_ascii(space, u, tc):
+    _unmarshal_ascii(True, False)
+ at unmarshaller(TYPE_SHORT_ASCII_INTERNED)
+def unmarshal_ascii(space, u, tc):
+    _unmarshal_ascii(True, True)
+
 
 @marshaller(W_SetObject)
 def marshal_set(space, w_set, m):

From pypy.commits at gmail.com  Sat Aug 27 14:49:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 11:49:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: fix
Message-ID: <57c1e09c.06b0c20a.5eba8.ca5f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86618:993d50cc614c
Date: 2016-08-27 20:47 +0200
http://bitbucket.org/pypy/pypy/changeset/993d50cc614c/

Log:	fix

diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -454,18 +454,29 @@
 def unmarshal_bytes(space, u, tc):
     return space.new_interned_str(u.get_str())
 
+def _unmarshal_ascii(u, short_length, interned):
+    if short_length:
+        lng = ord(u.get1())
+    else:
+        lng = u.get_lng()
+    s = u.get(lng)
+    w_u = u.space.newunicode(s.decode('latin1'))
+    if interned:
+        w_u = u.space.new_interned_w_str(w_u)
+    return w_u
+
 @unmarshaller(TYPE_ASCII)    # nb. never generated by pypy so far
 def unmarshal_ascii(space, u, tc):
-    _unmarshal_ascii(False, False)
+    return _unmarshal_ascii(u, False, False)
 @unmarshaller(TYPE_ASCII_INTERNED)
 def unmarshal_ascii(space, u, tc):
-    _unmarshal_ascii(False, True)
+    return _unmarshal_ascii(u, False, True)
 @unmarshaller(TYPE_SHORT_ASCII)
 def unmarshal_ascii(space, u, tc):
-    _unmarshal_ascii(True, False)
+    return _unmarshal_ascii(u, True, False)
 @unmarshaller(TYPE_SHORT_ASCII_INTERNED)
 def unmarshal_ascii(space, u, tc):
-    _unmarshal_ascii(True, True)
+    return _unmarshal_ascii(u, True, True)
 
 
 @marshaller(W_SetObject)

From pypy.commits at gmail.com  Sat Aug 27 14:51:55 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 Aug 2016 11:51:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Write class_attr() in a more idiomatic
 style
Message-ID: <57c1e14b.a699c20a.32011.770e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86619:5dc3b1a3241c
Date: 2016-08-27 19:51 +0100
http://bitbucket.org/pypy/pypy/changeset/5dc3b1a3241c/

Log:	Write class_attr() in a more idiomatic style

diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -1,6 +1,5 @@
 import pytest
 import sys
-from pypy.tool.pytest.objspace import gettestobjspace
 try:
     import __pypy__
 except ImportError:
@@ -23,22 +22,23 @@
 
 attrnames = strategies.sampled_from(["a", "b", "c"])
 
- at strategies.composite
-def class_attr(draw):
-    what = draw(strategies.sampled_from(["value", "method", "property"]))
-    if what == "value":
-        val = draw(strategies.integers())
-        return val, str(val)
-    if what == "method":
-        val = draw(strategies.integers())
-        return (lambda self, val=val: val,
-                "lambda self: %d" % val)
-    if what == "property":
-        val = draw(strategies.integers())
-        return (property(lambda self, val=val: val,
-                            lambda self, val: None,
-                            lambda self: None),
-                "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+def make_value_attr(val):
+    return val, str(val)
+
+def make_method(val):
+    return (lambda self, val=val: val,
+            "lambda self: %d" % val)
+
+def make_property(val):
+    return (
+        property(lambda self: val, lambda self, val: None, lambda self: None),
+        "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+value_attrs = strategies.builds(make_value_attr, strategies.integers())
+methods = strategies.builds(make_method, strategies.integers())
+properties = strategies.builds(make_property, strategies.integers())
+class_attrs = strategies.one_of(value_attrs, methods, properties)
+
 
 @strategies.composite
 def make_code(draw):
@@ -55,7 +55,7 @@
     for name in ["a", "b", "c"]:
         if not draw(strategies.booleans()):
             continue
-        dct[name], codeval = draw(class_attr())
+        dct[name], codeval = draw(class_attrs)
         code.append("    %s = %s" % (name, codeval))
     class OldBase: pass
     class NewBase(object): pass
@@ -99,11 +99,11 @@
             else:
                 code.append("a.%s = lambda : %s" % (attr, val))
         elif op == "writeclass":
-            val, codeval = draw(class_attr())
+            val, codeval = draw(class_attrs)
             setattr(cls, attr, val)
             code.append("A.%s = %s" % (attr, codeval))
         elif op == "writebase":
-            val, codeval = draw(class_attr())
+            val, codeval = draw(class_attrs)
             setattr(OldBase, attr, val)
             setattr(NewBase, attr, val)
             code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))

From pypy.commits at gmail.com  Sat Aug 27 15:01:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 12:01:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: Bump the magic number,
 to avoid using old .pyc files
Message-ID: <57c1e393.d42f1c0a.d3e46.aaff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86620:8590483012bd
Date: 2016-08-27 21:00 +0200
http://bitbucket.org/pypy/pypy/changeset/8590483012bd/

Log:	Bump the magic number, to avoid using old .pyc files

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -37,7 +37,7 @@
 # different value for the highest 16 bits. Bump pypy_incremental_magic every
 # time you make pyc files incompatible
 
-pypy_incremental_magic = 64 # bump it by 16
+pypy_incremental_magic = 80 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value

From pypy.commits at gmail.com  Sat Aug 27 15:06:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 12:06:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: Write a version-dependent cache
 here
Message-ID: <57c1e4c6.03121c0a.d683b.ac0c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86621:d663392aa622
Date: 2016-08-27 21:05 +0200
http://bitbucket.org/pypy/pypy/changeset/d663392aa622/

Log:	Write a version-dependent cache here

diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -47,8 +47,10 @@
     def _cached_compile(space, name, source, *args):
         from rpython.config.translationoption import CACHE_DIR
         from pypy.module.marshal import interp_marshal
+        from pypy.interpreter.pycode import default_magic
 
-        cachename = os.path.join(CACHE_DIR, 'frozen_importlib%s' % (name,))
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_%d%s' % (
+            default_magic, name))
         try:
             if space.config.translating:
                 raise IOError("don't use the cache when translating pypy")

From pypy.commits at gmail.com  Sat Aug 27 15:16:33 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 Aug 2016 12:16:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix typo
Message-ID: <57c1e711.c62f1c0a.57e03.ae33@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86622:2ae2489148f1
Date: 2016-08-27 20:15 +0100
http://bitbucket.org/pypy/pypy/changeset/2ae2489148f1/

Log:	fix typo

diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -198,7 +198,7 @@
     "buffer_test",
     "Module Doc",
     -1,
-    buffer_functions;
+    buffer_functions,
     NULL,
     NULL,
     NULL,

From pypy.commits at gmail.com  Sat Aug 27 15:17:36 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 Aug 2016 12:17:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: rm obsolete check
Message-ID: <57c1e750.11051c0a.563fc.b568@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86623:565fb0c787b6
Date: 2016-08-27 20:17 +0100
http://bitbucket.org/pypy/pypy/changeset/565fb0c787b6/

Log:	rm obsolete check

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,14 +1,9 @@
-import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
@@ -29,4 +24,3 @@
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)
-

From pypy.commits at gmail.com  Sat Aug 27 16:13:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 13:13:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: pfff
Message-ID: <57c1f474.cb7f1c0a.fa27d.c432@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86624:62948ab72ff7
Date: 2016-08-27 22:12 +0200
http://bitbucket.org/pypy/pypy/changeset/62948ab72ff7/

Log:	pfff

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -981,7 +981,7 @@
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
-    add_fork_hook('child', _reinit_tls)
+    add_fork_hook('child', reinit_tls)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)

From pypy.commits at gmail.com  Sat Aug 27 16:14:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 13:14:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c1f4a4.05d71c0a.351b2.6889@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86625:0d0c1fea2c5b
Date: 2016-08-27 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/0d0c1fea2c5b/

Log:	hg merge default

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -983,7 +983,7 @@
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
-    add_fork_hook('child', _reinit_tls)
+    add_fork_hook('child', reinit_tls)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -198,7 +198,7 @@
     "buffer_test",
     "Module Doc",
     -1,
-    buffer_functions;
+    buffer_functions,
     NULL,
     NULL,
     NULL,
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,13 +1,8 @@
-import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -1,6 +1,5 @@
 import pytest
 import sys
-from pypy.tool.pytest.objspace import gettestobjspace
 try:
     import __pypy__
 except ImportError:
@@ -23,22 +22,23 @@
 
 attrnames = strategies.sampled_from(["a", "b", "c"])
 
- at strategies.composite
-def class_attr(draw):
-    what = draw(strategies.sampled_from(["value", "method", "property"]))
-    if what == "value":
-        val = draw(strategies.integers())
-        return val, str(val)
-    if what == "method":
-        val = draw(strategies.integers())
-        return (lambda self, val=val: val,
-                "lambda self: %d" % val)
-    if what == "property":
-        val = draw(strategies.integers())
-        return (property(lambda self, val=val: val,
-                            lambda self, val: None,
-                            lambda self: None),
-                "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+def make_value_attr(val):
+    return val, str(val)
+
+def make_method(val):
+    return (lambda self, val=val: val,
+            "lambda self: %d" % val)
+
+def make_property(val):
+    return (
+        property(lambda self: val, lambda self, val: None, lambda self: None),
+        "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+value_attrs = strategies.builds(make_value_attr, strategies.integers())
+methods = strategies.builds(make_method, strategies.integers())
+properties = strategies.builds(make_property, strategies.integers())
+class_attrs = strategies.one_of(value_attrs, methods, properties)
+
 
 @strategies.composite
 def make_code(draw):
@@ -55,7 +55,7 @@
     for name in ["a", "b", "c"]:
         if not draw(strategies.booleans()):
             continue
-        dct[name], codeval = draw(class_attr())
+        dct[name], codeval = draw(class_attrs)
         code.append("    %s = %s" % (name, codeval))
     class OldBase: pass
     class NewBase(object): pass
@@ -99,11 +99,11 @@
             else:
                 code.append("a.%s = lambda : %s" % (attr, val))
         elif op == "writeclass":
-            val, codeval = draw(class_attr())
+            val, codeval = draw(class_attrs)
             setattr(cls, attr, val)
             code.append("A.%s = %s" % (attr, codeval))
         elif op == "writebase":
-            val, codeval = draw(class_attr())
+            val, codeval = draw(class_attrs)
             setattr(OldBase, attr, val)
             setattr(NewBase, attr, val)
             code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))

From pypy.commits at gmail.com  Sat Aug 27 16:14:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 13:14:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c1f4a6.919a1c0a.bca81.cd40@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86626:6833b14f183f
Date: 2016-08-27 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/6833b14f183f/

Log:	hg merge py3k

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -983,7 +983,7 @@
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
-    add_fork_hook('child', _reinit_tls)
+    add_fork_hook('child', reinit_tls)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -198,7 +198,7 @@
     "buffer_test",
     "Module Doc",
     -1,
-    buffer_functions;
+    buffer_functions,
     NULL,
     NULL,
     NULL,
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,13 +1,8 @@
-import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -1,6 +1,5 @@
 import pytest
 import sys
-from pypy.tool.pytest.objspace import gettestobjspace
 try:
     import __pypy__
 except ImportError:
@@ -23,22 +22,23 @@
 
 attrnames = strategies.sampled_from(["a", "b", "c"])
 
- at strategies.composite
-def class_attr(draw):
-    what = draw(strategies.sampled_from(["value", "method", "property"]))
-    if what == "value":
-        val = draw(strategies.integers())
-        return val, str(val)
-    if what == "method":
-        val = draw(strategies.integers())
-        return (lambda self, val=val: val,
-                "lambda self: %d" % val)
-    if what == "property":
-        val = draw(strategies.integers())
-        return (property(lambda self, val=val: val,
-                            lambda self, val: None,
-                            lambda self: None),
-                "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+def make_value_attr(val):
+    return val, str(val)
+
+def make_method(val):
+    return (lambda self, val=val: val,
+            "lambda self: %d" % val)
+
+def make_property(val):
+    return (
+        property(lambda self: val, lambda self, val: None, lambda self: None),
+        "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+value_attrs = strategies.builds(make_value_attr, strategies.integers())
+methods = strategies.builds(make_method, strategies.integers())
+properties = strategies.builds(make_property, strategies.integers())
+class_attrs = strategies.one_of(value_attrs, methods, properties)
+
 
 @strategies.composite
 def make_code(draw):
@@ -55,7 +55,7 @@
     for name in ["a", "b", "c"]:
         if not draw(strategies.booleans()):
             continue
-        dct[name], codeval = draw(class_attr())
+        dct[name], codeval = draw(class_attrs)
         code.append("    %s = %s" % (name, codeval))
     class OldBase: pass
     class NewBase(object): pass
@@ -99,11 +99,11 @@
             else:
                 code.append("a.%s = lambda : %s" % (attr, val))
         elif op == "writeclass":
-            val, codeval = draw(class_attr())
+            val, codeval = draw(class_attrs)
             setattr(cls, attr, val)
             code.append("A.%s = %s" % (attr, codeval))
         elif op == "writebase":
-            val, codeval = draw(class_attr())
+            val, codeval = draw(class_attrs)
             setattr(OldBase, attr, val)
             setattr(NewBase, attr, val)
             code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))

From pypy.commits at gmail.com  Sat Aug 27 16:14:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 Aug 2016 13:14:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: hg merge py3.5
Message-ID: <57c1f4a8.45c8c20a.e6dc0.9506@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86627:7ff600e2853d
Date: 2016-08-27 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/7ff600e2853d/

Log:	hg merge py3.5

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -983,7 +983,7 @@
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
-    add_fork_hook('child', _reinit_tls)
+    add_fork_hook('child', reinit_tls)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -198,7 +198,7 @@
     "buffer_test",
     "Module Doc",
     -1,
-    buffer_functions;
+    buffer_functions,
     NULL,
     NULL,
     NULL,
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,13 +1,8 @@
-import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -1,6 +1,5 @@
 import pytest
 import sys
-from pypy.tool.pytest.objspace import gettestobjspace
 try:
     import __pypy__
 except ImportError:
@@ -23,22 +22,23 @@
 
 attrnames = strategies.sampled_from(["a", "b", "c"])
 
- at strategies.composite
-def class_attr(draw):
-    what = draw(strategies.sampled_from(["value", "method", "property"]))
-    if what == "value":
-        val = draw(strategies.integers())
-        return val, str(val)
-    if what == "method":
-        val = draw(strategies.integers())
-        return (lambda self, val=val: val,
-                "lambda self: %d" % val)
-    if what == "property":
-        val = draw(strategies.integers())
-        return (property(lambda self, val=val: val,
-                            lambda self, val: None,
-                            lambda self: None),
-                "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+def make_value_attr(val):
+    return val, str(val)
+
+def make_method(val):
+    return (lambda self, val=val: val,
+            "lambda self: %d" % val)
+
+def make_property(val):
+    return (
+        property(lambda self: val, lambda self, val: None, lambda self: None),
+        "property(lambda self: %d, lambda self, val: None, lambda self: None)" % val)
+
+value_attrs = strategies.builds(make_value_attr, strategies.integers())
+methods = strategies.builds(make_method, strategies.integers())
+properties = strategies.builds(make_property, strategies.integers())
+class_attrs = strategies.one_of(value_attrs, methods, properties)
+
 
 @strategies.composite
 def make_code(draw):
@@ -55,7 +55,7 @@
     for name in ["a", "b", "c"]:
         if not draw(strategies.booleans()):
             continue
-        dct[name], codeval = draw(class_attr())
+        dct[name], codeval = draw(class_attrs)
         code.append("    %s = %s" % (name, codeval))
     class OldBase: pass
     class NewBase(object): pass
@@ -99,11 +99,11 @@
             else:
                 code.append("a.%s = lambda : %s" % (attr, val))
         elif op == "writeclass":
-            val, codeval = draw(class_attr())
+            val, codeval = draw(class_attrs)
             setattr(cls, attr, val)
             code.append("A.%s = %s" % (attr, codeval))
         elif op == "writebase":
-            val, codeval = draw(class_attr())
+            val, codeval = draw(class_attrs)
             setattr(OldBase, attr, val)
             setattr(NewBase, attr, val)
             code.append("OldBase.%s = NewBase.%s = %s" % (attr, attr , codeval))

From pypy.commits at gmail.com  Sat Aug 27 21:11:29 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 27 Aug 2016 18:11:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: correct divergence-from-release hash
Message-ID: <57c23a41.031dc20a.f5e91.d96f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86628:9845c6c7144b
Date: 2016-08-28 11:06 +1000
http://bitbucket.org/pypy/pypy/changeset/9845c6c7144b/

Log:	correct divergence-from-release hash

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,6 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 4176c6f63109
+.. startrev: 522736f816dc
 
 

From pypy.commits at gmail.com  Sat Aug 27 21:11:31 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 27 Aug 2016 18:11:31 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: add failing test for new
 buffer interface PyMemoryView_GET_BUFFER
Message-ID: <57c23a43.436ec20a.d0586.dd5e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86629:e9a0c626b91a
Date: 2016-08-28 11:09 +1000
http://bitbucket.org/pypy/pypy/changeset/e9a0c626b91a/

Log:	add failing test for new buffer interface PyMemoryView_GET_BUFFER

diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
   PyMyArray* self = (PyMyArray*)obj;
-  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
-    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
-    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
@@ -188,7 +185,23 @@
     (initproc)PyMyArray_init,     /* tp_init */
 };
 
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+    Py_buffer* view = NULL;
+    PyObject* obj = PyTuple_GetItem(args, 0);
+    PyObject* memoryview = PyMemoryView_FromObject(obj);
+    if (memoryview == NULL)
+        return PyInt_FromLong(-1);
+    view = PyMemoryView_GET_BUFFER(memoryview);
+    Py_DECREF(memoryview);
+    return PyInt_FromLong(view->len);
+}
+
+
+
 static PyMethodDef buffer_functions[] = {
+    {"test_buffer",   (PyCFunction)test_buffer, METH_VARARGS, NULL},
     {NULL,        NULL}    /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -24,3 +24,5 @@
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)
+        viewlen = module.test_buffer(arr)
+        assert viewlen == y.itemsize * len(y)

From pypy.commits at gmail.com  Sat Aug 27 21:54:43 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 Aug 2016 18:54:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Resynchronise rpython/ with branch 'py3.5'
Message-ID: <57c24463.a710c20a.54e5.e2a9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86630:991f4c162891
Date: 2016-08-28 02:53 +0100
http://bitbucket.org/pypy/pypy/changeset/991f4c162891/

Log:	Resynchronise rpython/ with branch 'py3.5'

diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -176,6 +176,7 @@
 
 
 SOCK_DGRAM SOCK_RAW SOCK_RDM SOCK_SEQPACKET SOCK_STREAM
+SOCK_CLOEXEC
 
 SOL_SOCKET SOL_IPX SOL_AX25 SOL_ATALK SOL_NETROM SOL_ROSE
 
@@ -319,6 +320,8 @@
                                           [('p_proto', rffi.INT),
                                            ])
 
+CConfig.HAVE_ACCEPT4 = platform.Has('accept4')
+
 if _POSIX:
     CConfig.nfds_t = platform.SimpleType('nfds_t')
     CConfig.pollfd = platform.Struct('struct pollfd',
@@ -541,6 +544,12 @@
 socketaccept = external('accept', [socketfd_type, sockaddr_ptr,
                                    socklen_t_ptr], socketfd_type,
                         save_err=SAVE_ERR)
+HAVE_ACCEPT4 = cConfig.HAVE_ACCEPT4
+if HAVE_ACCEPT4:
+    socketaccept4 = external('accept4', [socketfd_type, sockaddr_ptr,
+                                         socklen_t_ptr, rffi.INT],
+                                        socketfd_type,
+                             save_err=SAVE_ERR)
 socketbind = external('bind', [socketfd_type, sockaddr_ptr, socklen_t],
                               rffi.INT, save_err=SAVE_ERR)
 socketlisten = external('listen', [socketfd_type, rffi.INT], rffi.INT,
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -372,15 +372,27 @@
         raise OSError(get_saved_errno(), '%s failed' % name)
     return result
 
+def _dup(fd, inheritable=True):
+    validate_fd(fd)
+    if inheritable:
+        res = c_dup(fd)
+    else:
+        res = c_dup_noninheritable(fd)
+    return res
+
 @replace_os_function('dup')
-def dup(fd):
-    validate_fd(fd)
-    return handle_posix_error('dup', c_dup(fd))
+def dup(fd, inheritable=True):
+    res = _dup(fd, inheritable)
+    return handle_posix_error('dup', res)
 
 @replace_os_function('dup2')
-def dup2(fd, newfd):
+def dup2(fd, newfd, inheritable=True):
     validate_fd(fd)
-    handle_posix_error('dup2', c_dup2(fd, newfd))
+    if inheritable:
+        res = c_dup2(fd, newfd)
+    else:
+        res = c_dup2_noninheritable(fd, newfd)
+    handle_posix_error('dup2', res)
 
 #___________________________________________________________________
 
@@ -1122,37 +1134,77 @@
     c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
                                                     rffi.INT],
                                 rffi.INT)
+    HAVE_PIPE2 = False
+    HAVE_DUP3 = False
+    O_CLOEXEC = None
 else:
     INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
     c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT,
                       save_err=rffi.RFFI_SAVE_ERRNO)
+    class CConfig:
+        _compilation_info_ = eci
+        HAVE_PIPE2 = rffi_platform.Has('pipe2')
+        HAVE_DUP3 = rffi_platform.Has('dup3')
+        O_CLOEXEC = rffi_platform.DefinedConstantInteger('O_CLOEXEC')
+    config = rffi_platform.configure(CConfig)
+    HAVE_PIPE2 = config['HAVE_PIPE2']
+    HAVE_DUP3 = config['HAVE_DUP3']
+    O_CLOEXEC = config['O_CLOEXEC']
+    if HAVE_PIPE2:
+        c_pipe2 = external('pipe2', [INT_ARRAY_P, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('pipe')
-def pipe():
+def pipe(flags=0):
+    # 'flags' might be ignored.  Check the result.
     if _WIN32:
+        # 'flags' ignored
         pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         try:
-            if not CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                raise WindowsError(rwin32.GetLastError_saved(),
-                                   "CreatePipe failed")
+            ok = CreatePipe(
+                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0)
             hread = rffi.cast(rffi.INTPTR_T, pread[0])
             hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
         finally:
             lltype.free(pwrite, flavor='raw')
             lltype.free(pread, flavor='raw')
-        fdread = c_open_osfhandle(hread, 0)
-        fdwrite = c_open_osfhandle(hwrite, 1)
+        if ok:
+            fdread = c_open_osfhandle(hread, 0)
+            fdwrite = c_open_osfhandle(hwrite, 1)
+            if fdread == -1 or fdwrite == -1:
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
+                ok = 0
+        if not ok:
+            raise WindowsError(rwin32.GetLastError_saved(),
+                               "CreatePipe failed")
         return (fdread, fdwrite)
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
-            handle_posix_error('pipe', c_pipe(filedes))
+            if HAVE_PIPE2 and _pipe2_syscall.attempt_syscall():
+                res = c_pipe2(filedes, flags)
+                if _pipe2_syscall.fallback(res):
+                    res = c_pipe(filedes)
+            else:
+                res = c_pipe(filedes)      # 'flags' ignored
+            handle_posix_error('pipe', res)
             return (widen(filedes[0]), widen(filedes[1]))
         finally:
             lltype.free(filedes, flavor='raw')
 
+def pipe2(flags):
+    # Only available if there is really a c_pipe2 function.
+    # No fallback to pipe() if we get ENOSYS.
+    filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
+    try:
+        res = c_pipe2(filedes, flags)
+        handle_posix_error('pipe2', res)
+        return (widen(filedes[0]), widen(filedes[1]))
+    finally:
+        lltype.free(filedes, flavor='raw')
+
 c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
                   save_err=rffi.RFFI_SAVE_ERRNO,)
 c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
@@ -2088,14 +2140,46 @@
 
 
 eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=["""
+    separate_module_sources=[r"""
+#include <errno.h>
+
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)
 {
-    /* XXX minimal impl. XXX */
-    int request = inheritable ? FIONCLEX : FIOCLEX;
-    return ioctl(fd, request, NULL);
+    static int ioctl_works = -1;
+    int flags;
+
+    if (ioctl_works != 0) {
+        int request = inheritable ? FIONCLEX : FIOCLEX;
+        int err = ioctl(fd, request, NULL);
+        if (!err) {
+            ioctl_works = 1;
+            return 0;
+        }
+
+        if (errno != ENOTTY && errno != EACCES) {
+            return -1;
+        }
+        else {
+            /* ENOTTY: The ioctl is declared but not supported by the
+               kernel.  EACCES: SELinux policy, this can be the case on
+               Android. */
+            ioctl_works = 0;
+        }
+        /* fallback to fcntl() if ioctl() does not work */
+    }
+
+    flags = fcntl(fd, F_GETFD);
+    if (flags < 0)
+        return -1;
+
+    if (inheritable)
+        flags &= ~FD_CLOEXEC;
+    else
+        flags |= FD_CLOEXEC;
+    return fcntl(fd, F_SETFD, flags);
 }
+
 RPY_EXTERN
 int rpy_get_inheritable(int fd)
 {
@@ -2104,8 +2188,64 @@
         return -1;
     return !(flags & FD_CLOEXEC);
 }
-    """],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+RPY_EXTERN
+int rpy_dup_noninheritable(int fd)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUPFD_CLOEXEC
+    return fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+    fd = dup(fd);
+    if (fd >= 0) {
+        if (rpy_set_inheritable(fd, 0) != 0) {
+            close(fd);
+            return -1;
+        }
+    }
+    return fd;
+#endif
+}
+
+RPY_EXTERN
+int rpy_dup2_noninheritable(int fd, int fd2)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUP2FD_CLOEXEC
+    return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
+
+#else
+# if %(HAVE_DUP3)d   /* HAVE_DUP3 */
+    static int dup3_works = -1;
+    if (dup3_works != 0) {
+        if (dup3(fd, fd2, O_CLOEXEC) >= 0)
+            return 0;
+        if (dup3_works == -1)
+            dup3_works = (errno != ENOSYS);
+        if (dup3_works)
+            return -1;
+    }
+# endif
+    if (dup2(fd, fd2) < 0)
+        return -1;
+    if (rpy_set_inheritable(fd2, 0) != 0) {
+        close(fd2);
+        return -1;
+    }
+    return 0;
+#endif
+}
+    """ % {'HAVE_DUP3': HAVE_DUP3}],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
 
 c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
@@ -2113,12 +2253,56 @@
 c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
                              compilation_info=eci_inheritable)
+c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                compilation_info=eci_inheritable)
+c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                 compilation_info=eci_inheritable)
 
 def set_inheritable(fd, inheritable):
-    error = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', error)
+    result = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', result)
 
 def get_inheritable(fd):
     res = c_get_inheritable(fd)
     res = handle_posix_error('get_inheritable', res)
     return res != 0
+
+class SetNonInheritableCache(object):
+    """Make one prebuilt instance of this for each path that creates
+    file descriptors, where you don't necessarily know if that function
+    returns inheritable or non-inheritable file descriptors.
+    """
+    _immutable_fields_ = ['cached_inheritable?']
+    cached_inheritable = -1    # -1 = don't know yet; 0 = off; 1 = on
+
+    def set_non_inheritable(self, fd):
+        if self.cached_inheritable == -1:
+            self.cached_inheritable = get_inheritable(fd)
+        if self.cached_inheritable == 1:
+            # 'fd' is inheritable; we must manually turn it off
+            set_inheritable(fd, False)
+
+    def _cleanup_(self):
+        self.cached_inheritable = -1
+
+class ENoSysCache(object):
+    """Cache whether a system call returns ENOSYS or not."""
+    _immutable_fields_ = ['cached_nosys?']
+    cached_nosys = -1      # -1 = don't know; 0 = no; 1 = yes, getting ENOSYS
+
+    def attempt_syscall(self):
+        return self.cached_nosys != 1
+
+    def fallback(self, res):
+        nosys = self.cached_nosys
+        if nosys == -1:
+            nosys = (res < 0 and get_saved_errno() == errno.ENOSYS)
+            self.cached_nosys = nosys
+        return nosys
+
+    def _cleanup_(self):
+        self.cached_nosys = -1
+
+_pipe2_syscall = ENoSysCache()
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -8,10 +8,11 @@
 # XXX this does not support yet the least common AF_xxx address families
 # supported by CPython.  See http://bugs.pypy.org/issue1942
 
+from errno import EINVAL
 from rpython.rlib import _rsocket_rffi as _c, jit, rgc
 from rpython.rlib.objectmodel import instantiate, keepalive_until_here
 from rpython.rlib.rarithmetic import intmask, r_uint
-from rpython.rlib import rthread
+from rpython.rlib import rthread, rposix
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rffi import sizeof, offsetof
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -522,12 +523,28 @@
     timeout = -1.0
 
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0,
-                 fd=_c.INVALID_SOCKET):
+                 fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            fd = _c.socket(family, type, proto)
-        if _c.invalid_socket(fd):
-            raise self.error_handler()
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socket() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                fd = _c.socket(family, type | SOCK_CLOEXEC, proto)
+                if fd < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise self.error_handler()
+            if _c.invalid_socket(fd):
+                fd = _c.socket(family, type, proto)
+                if _c.invalid_socket(fd):
+                    raise self.error_handler()
+                if not inheritable:
+                    sock_set_inheritable(fd, False)
         # PLAT RISCOS
         self.fd = fd
         self.family = family
@@ -630,20 +647,33 @@
         return addr, addr.addr_p, addrlen_p
 
     @jit.dont_look_inside
-    def accept(self):
+    def accept(self, inheritable=True):
         """Wait for an incoming connection.
         Return (new socket fd, client address)."""
         if self._select(False) == 1:
             raise SocketTimeout
         address, addr_p, addrlen_p = self._addrbuf()
         try:
-            newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+            remove_inheritable = not inheritable
+            if (not inheritable and SOCK_CLOEXEC is not None
+                    and _c.HAVE_ACCEPT4
+                    and _accept4_syscall.attempt_syscall()):
+                newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
+                                         SOCK_CLOEXEC)
+                if _accept4_syscall.fallback(newfd):
+                    newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+                else:
+                    remove_inheritable = False
+            else:
+                newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
             addrlen = addrlen_p[0]
         finally:
             lltype.free(addrlen_p, flavor='raw')
             address.unlock()
         if _c.invalid_socket(newfd):
             raise self.error_handler()
+        if remove_inheritable:
+            sock_set_inheritable(newfd, False)
         address.addrlen = rffi.cast(lltype.Signed, addrlen)
         return (newfd, address)
 
@@ -1032,6 +1062,12 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
+def sock_set_inheritable(fd, inheritable):
+    try:
+        rposix.set_inheritable(fd, inheritable)
+    except OSError as e:
+        raise CSocketError(e.errno)
+
 class SocketError(Exception):
     applevelerrcls = 'error'
     def __init__(self):
@@ -1090,7 +1126,7 @@
 
 if hasattr(_c, 'socketpair'):
     def socketpair(family=socketpair_default_family, type=SOCK_STREAM, proto=0,
-                   SocketClass=RSocket):
+                   SocketClass=RSocket, inheritable=True):
         """socketpair([family[, type[, proto]]]) -> (socket object, socket object)
 
         Create a pair of socket objects from the sockets returned by the platform
@@ -1100,18 +1136,41 @@
         """
         result = lltype.malloc(_c.socketpair_t, 2, flavor='raw')
         try:
-            res = _c.socketpair(family, type, proto, result)
+            res = -1
+            remove_inheritable = not inheritable
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socketpair() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                res = _c.socketpair(family, type | SOCK_CLOEXEC,
+                                    proto, result)
+                if res < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise last_error()
+                else:
+                    remove_inheritable = False
+            #
             if res < 0:
-                raise last_error()
+                res = _c.socketpair(family, type, proto, result)
+                if res < 0:
+                    raise last_error()
             fd0 = rffi.cast(lltype.Signed, result[0])
             fd1 = rffi.cast(lltype.Signed, result[1])
         finally:
             lltype.free(result, flavor='raw')
+        if remove_inheritable:
+            sock_set_inheritable(fd0, False)
+            sock_set_inheritable(fd1, False)
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
 if _c.WIN32:
-    def dup(fd):
+    def dup(fd, inheritable=True):
         with lltype.scoped_alloc(_c.WSAPROTOCOL_INFO, zero=True) as info:
             if _c.WSADuplicateSocket(fd, rwin32.GetCurrentProcessId(), info):
                 raise last_error()
@@ -1122,15 +1181,16 @@
                 raise last_error()
             return result
 else:
-    def dup(fd):
-        fd = _c.dup(fd)
+    def dup(fd, inheritable=True):
+        fd = rposix._dup(fd, inheritable)
         if fd < 0:
             raise last_error()
         return fd
 
-def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
+def fromfd(fd, family, type, proto=0, SocketClass=RSocket, inheritable=True):
     # Dup the fd so it and the socket can be closed independently
-    return make_socket(dup(fd), family, type, proto, SocketClass)
+    fd = dup(fd, inheritable=inheritable)
+    return make_socket(fd, family, type, proto, SocketClass)
 
 def getdefaulttimeout():
     return defaults.timeout
@@ -1407,3 +1467,5 @@
     if timeout < 0.0:
         timeout = -1.0
     defaults.timeout = timeout
+
+_accept4_syscall = rposix.ENoSysCache()
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -24,12 +24,15 @@
 #### Constants
 
 # Identifying as _sre from Python 2.3 and onwards (at least up to 2.7)
-MAGIC = 20031017
+# UPDATE: change was necessary for Python 3.3 changes
+MAGIC = 20140917
 
 if sys.maxint > 2**32:
     MAXREPEAT = int(2**32 - 1)
+    MAXGROUPS = int(2**31 - 1)
 else:
     MAXREPEAT = int(2**31 - 1)
+    MAXGROUPS = int((2**31 / sys.maxint / 2) - 1)
 
 # In _sre.c this is bytesize of the code word type of the C implementation.
 # There it's 2 for normal Python builds and more for wide unicode builds (large 
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -589,3 +589,18 @@
     assert rposix.get_inheritable(fd1) == False
     os.close(fd1)
     os.close(fd2)
+
+def test_SetNonInheritableCache():
+    cache = rposix.SetNonInheritableCache()
+    fd1, fd2 = os.pipe()
+    assert rposix.get_inheritable(fd1) == True
+    assert rposix.get_inheritable(fd1) == True
+    assert cache.cached_inheritable == -1
+    cache.set_non_inheritable(fd1)
+    assert cache.cached_inheritable == 1
+    cache.set_non_inheritable(fd2)
+    assert cache.cached_inheritable == 1
+    assert rposix.get_inheritable(fd1) == False
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -119,6 +119,16 @@
     s1.close()
     s2.close()
 
+def test_socketpair_inheritable():
+    if sys.platform == "win32":
+        py.test.skip('No socketpair on Windows')
+    for inh in [False, True]:
+        s1, s2 = socketpair(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        assert rposix.get_inheritable(s2.fd) == inh
+        s1.close()
+        s2.close()
+
 def test_socketpair_recvinto_1():
     class Buffer:
         def setslice(self, start, string):
@@ -378,6 +388,12 @@
     s1.close()
     s2.close()
 
+def test_inheritable():
+    for inh in [False, True]:
+        s1 = RSocket(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        s1.close()
+
 def test_getaddrinfo_http():
     lst = getaddrinfo('localhost', 'http')
     assert isinstance(lst, list)
diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -58,7 +58,7 @@
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
     expect(f, g, "ll_os.ll_os_open", ("/tmp/foobar", os.O_RDONLY, 0777), 77)
-    expect(f, g, "ll_os.ll_os_dup",  (77,), 78)
+    expect(f, g, "ll_os.ll_os_dup",  (77, True), 78)
     g.close()
     tail = f.read()
     f.close()
@@ -94,7 +94,7 @@
 
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
-    expect(f, g, "ll_os.ll_os_dup2",   (34, 56), None)
+    expect(f, g, "ll_os.ll_os_dup2",   (34, 56, True), None)
     expect(f, g, "ll_os.ll_os_access", ("spam", 77), True)
     g.close()
     tail = f.read()
@@ -134,7 +134,7 @@
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
     expect(f, g, "ll_time.ll_time_time", (), 3.141592)
-    expect(f, g, "ll_os.ll_os_dup", (3141,), 3)
+    expect(f, g, "ll_os.ll_os_dup", (3141, True), 3)
     g.close()
     tail = f.read()
     f.close()
@@ -149,7 +149,7 @@
     exe = compile(entry_point)
     g, f = run_in_subprocess(exe)
     expect(f, g, "ll_os.ll_os_getcwd", (), "/tmp/foo/bar")
-    expect(f, g, "ll_os.ll_os_dup", (len("/tmp/foo/bar"),), 3)
+    expect(f, g, "ll_os.ll_os_dup", (len("/tmp/foo/bar"), True), 3)
     g.close()
     tail = f.read()
     f.close()

From pypy.commits at gmail.com  Sat Aug 27 22:02:59 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 Aug 2016 19:02:59 -0700 (PDT)
Subject: [pypy-commit] pypy rpython-resync: Resynchronise rpython/ with
 branch 'py3k'
Message-ID: <57c24653.109a1c0a.c338b.1d80@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rpython-resync
Changeset: r86631:7d051416b2f1
Date: 2016-08-28 03:01 +0100
http://bitbucket.org/pypy/pypy/changeset/7d051416b2f1/

Log:	Resynchronise rpython/ with branch 'py3k'

diff too long, truncating to 2000 out of 208757 lines

diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -176,6 +176,7 @@
 
 
 SOCK_DGRAM SOCK_RAW SOCK_RDM SOCK_SEQPACKET SOCK_STREAM
+SOCK_CLOEXEC
 
 SOL_SOCKET SOL_IPX SOL_AX25 SOL_ATALK SOL_NETROM SOL_ROSE
 
@@ -319,6 +320,8 @@
                                           [('p_proto', rffi.INT),
                                            ])
 
+CConfig.HAVE_ACCEPT4 = platform.Has('accept4')
+
 if _POSIX:
     CConfig.nfds_t = platform.SimpleType('nfds_t')
     CConfig.pollfd = platform.Struct('struct pollfd',
@@ -541,6 +544,12 @@
 socketaccept = external('accept', [socketfd_type, sockaddr_ptr,
                                    socklen_t_ptr], socketfd_type,
                         save_err=SAVE_ERR)
+HAVE_ACCEPT4 = cConfig.HAVE_ACCEPT4
+if HAVE_ACCEPT4:
+    socketaccept4 = external('accept4', [socketfd_type, sockaddr_ptr,
+                                         socklen_t_ptr, rffi.INT],
+                                        socketfd_type,
+                             save_err=SAVE_ERR)
 socketbind = external('bind', [socketfd_type, sockaddr_ptr, socklen_t],
                               rffi.INT, save_err=SAVE_ERR)
 socketlisten = external('listen', [socketfd_type, rffi.INT], rffi.INT,
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -10,6 +10,7 @@
     _immutable_ = True
 
     def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
         raise NotImplementedError
 
     def __len__(self):
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1222,6 +1222,9 @@
         # base is supposed to be positive or 0.0, which means we use e
         if base == 10.0:
             return _loghelper(math.log10, self)
+        if base == 2.0:
+            from rpython.rlib import rfloat
+            return _loghelper(rfloat.log2, self)
         ret = _loghelper(math.log, self)
         if base != 0.0:
             ret /= math.log(base)
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -126,6 +126,10 @@
         "SSL_OP_NO_COMPRESSION")
     SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger(
         "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS")
+    SSL_OP_CIPHER_SERVER_PREFERENCE = rffi_platform.ConstantInteger(
+        "SSL_OP_CIPHER_SERVER_PREFERENCE")
+    SSL_OP_SINGLE_DH_USE = rffi_platform.ConstantInteger(
+        "SSL_OP_SINGLE_DH_USE")
     HAS_SNI = rffi_platform.Defined("SSL_CTRL_SET_TLSEXT_HOSTNAME")
     HAS_NPN = rffi_platform.Defined("OPENSSL_NPN_NEGOTIATED")
     SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE")
@@ -307,6 +311,8 @@
 
 if HAVE_OPENSSL_RAND:
     ssl_external('RAND_add', [rffi.CCHARP, rffi.INT, rffi.DOUBLE], lltype.Void)
+    ssl_external('RAND_bytes', [rffi.UCHARP, rffi.INT], rffi.INT)
+    ssl_external('RAND_pseudo_bytes', [rffi.UCHARP, rffi.INT], rffi.INT)
     ssl_external('RAND_status', [], rffi.INT)
     if HAVE_OPENSSL_RAND_EGD:
         ssl_external('RAND_egd', [rffi.CCHARP], rffi.INT)
@@ -465,6 +471,7 @@
 ssl_external('GENERAL_NAME_print', [BIO, GENERAL_NAME], rffi.INT)
 ssl_external('pypy_GENERAL_NAME_dirn', [GENERAL_NAME], X509_NAME,
              macro=True)
+
 ssl_external('pypy_GENERAL_NAME_uri', [GENERAL_NAME], ASN1_IA5STRING,
              macro=True)
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -223,7 +223,7 @@
             pass
 
 if _WIN32:
-    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h']
+    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h']
     libraries = []
 else:
     if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')):
@@ -254,10 +254,11 @@
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
+    CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
-        CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
+        TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),
@@ -265,6 +266,12 @@
                            ('tms_cutime', rffi.INT),
                            ('tms_cstime', rffi.INT)])
 
+        WINSIZE = rffi_platform.Struct(
+            'struct winsize', [('ws_row', rffi.USHORT),
+                           ('ws_col', rffi.USHORT),
+                           ('ws_xpixel', rffi.USHORT),
+                           ('ws_ypixel', rffi.USHORT)])
+
     GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
     SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")
 
@@ -365,15 +372,27 @@
         raise OSError(get_saved_errno(), '%s failed' % name)
     return result
 
+def _dup(fd, inheritable=True):
+    validate_fd(fd)
+    if inheritable:
+        res = c_dup(fd)
+    else:
+        res = c_dup_noninheritable(fd)
+    return res
+
 @replace_os_function('dup')
-def dup(fd):
-    validate_fd(fd)
-    return handle_posix_error('dup', c_dup(fd))
+def dup(fd, inheritable=True):
+    res = _dup(fd, inheritable)
+    return handle_posix_error('dup', res)
 
 @replace_os_function('dup2')
-def dup2(fd, newfd):
+def dup2(fd, newfd, inheritable=True):
     validate_fd(fd)
-    handle_posix_error('dup2', c_dup2(fd, newfd))
+    if inheritable:
+        res = c_dup2(fd, newfd)
+    else:
+        res = c_dup2_noninheritable(fd, newfd)
+    handle_posix_error('dup2', res)
 
 #___________________________________________________________________
 
@@ -628,6 +647,8 @@
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
     c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+    c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 else:
     dirent_config = {}
 
@@ -1113,37 +1134,77 @@
     c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
                                                     rffi.INT],
                                 rffi.INT)
+    HAVE_PIPE2 = False
+    HAVE_DUP3 = False
+    O_CLOEXEC = None
 else:
     INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
     c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT,
                       save_err=rffi.RFFI_SAVE_ERRNO)
+    class CConfig:
+        _compilation_info_ = eci
+        HAVE_PIPE2 = rffi_platform.Has('pipe2')
+        HAVE_DUP3 = rffi_platform.Has('dup3')
+        O_CLOEXEC = rffi_platform.DefinedConstantInteger('O_CLOEXEC')
+    config = rffi_platform.configure(CConfig)
+    HAVE_PIPE2 = config['HAVE_PIPE2']
+    HAVE_DUP3 = config['HAVE_DUP3']
+    O_CLOEXEC = config['O_CLOEXEC']
+    if HAVE_PIPE2:
+        c_pipe2 = external('pipe2', [INT_ARRAY_P, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('pipe')
-def pipe():
+def pipe(flags=0):
+    # 'flags' might be ignored.  Check the result.
     if _WIN32:
+        # 'flags' ignored
         pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         try:
-            if not CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                raise WindowsError(rwin32.GetLastError_saved(),
-                                   "CreatePipe failed")
+            ok = CreatePipe(
+                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0)
             hread = rffi.cast(rffi.INTPTR_T, pread[0])
             hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
         finally:
             lltype.free(pwrite, flavor='raw')
             lltype.free(pread, flavor='raw')
-        fdread = c_open_osfhandle(hread, 0)
-        fdwrite = c_open_osfhandle(hwrite, 1)
+        if ok:
+            fdread = c_open_osfhandle(hread, 0)
+            fdwrite = c_open_osfhandle(hwrite, 1)
+            if fdread == -1 or fdwrite == -1:
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
+                ok = 0
+        if not ok:
+            raise WindowsError(rwin32.GetLastError_saved(),
+                               "CreatePipe failed")
         return (fdread, fdwrite)
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
-            handle_posix_error('pipe', c_pipe(filedes))
+            if HAVE_PIPE2 and _pipe2_syscall.attempt_syscall():
+                res = c_pipe2(filedes, flags)
+                if _pipe2_syscall.fallback(res):
+                    res = c_pipe(filedes)
+            else:
+                res = c_pipe(filedes)      # 'flags' ignored
+            handle_posix_error('pipe', res)
             return (widen(filedes[0]), widen(filedes[1]))
         finally:
             lltype.free(filedes, flavor='raw')
 
+def pipe2(flags):
+    # Only available if there is really a c_pipe2 function.
+    # No fallback to pipe() if we get ENOSYS.
+    filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
+    try:
+        res = c_pipe2(filedes, flags)
+        handle_posix_error('pipe2', res)
+        return (widen(filedes[0]), widen(filedes[1]))
+    finally:
+        lltype.free(filedes, flavor='raw')
+
 c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
                   save_err=rffi.RFFI_SAVE_ERRNO,)
 c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
@@ -2079,14 +2140,46 @@
 
 
 eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=["""
+    separate_module_sources=[r"""
+#include <errno.h>
+
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)
 {
-    /* XXX minimal impl. XXX */
-    int request = inheritable ? FIONCLEX : FIOCLEX;
-    return ioctl(fd, request, NULL);
+    static int ioctl_works = -1;
+    int flags;
+
+    if (ioctl_works != 0) {
+        int request = inheritable ? FIONCLEX : FIOCLEX;
+        int err = ioctl(fd, request, NULL);
+        if (!err) {
+            ioctl_works = 1;
+            return 0;
+        }
+
+        if (errno != ENOTTY && errno != EACCES) {
+            return -1;
+        }
+        else {
+            /* ENOTTY: The ioctl is declared but not supported by the
+               kernel.  EACCES: SELinux policy, this can be the case on
+               Android. */
+            ioctl_works = 0;
+        }
+        /* fallback to fcntl() if ioctl() does not work */
+    }
+
+    flags = fcntl(fd, F_GETFD);
+    if (flags < 0)
+        return -1;
+
+    if (inheritable)
+        flags &= ~FD_CLOEXEC;
+    else
+        flags |= FD_CLOEXEC;
+    return fcntl(fd, F_SETFD, flags);
 }
+
 RPY_EXTERN
 int rpy_get_inheritable(int fd)
 {
@@ -2095,8 +2188,64 @@
         return -1;
     return !(flags & FD_CLOEXEC);
 }
-    """],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+RPY_EXTERN
+int rpy_dup_noninheritable(int fd)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUPFD_CLOEXEC
+    return fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+    fd = dup(fd);
+    if (fd >= 0) {
+        if (rpy_set_inheritable(fd, 0) != 0) {
+            close(fd);
+            return -1;
+        }
+    }
+    return fd;
+#endif
+}
+
+RPY_EXTERN
+int rpy_dup2_noninheritable(int fd, int fd2)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUP2FD_CLOEXEC
+    return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
+
+#else
+# if %(HAVE_DUP3)d   /* HAVE_DUP3 */
+    static int dup3_works = -1;
+    if (dup3_works != 0) {
+        if (dup3(fd, fd2, O_CLOEXEC) >= 0)
+            return 0;
+        if (dup3_works == -1)
+            dup3_works = (errno != ENOSYS);
+        if (dup3_works)
+            return -1;
+    }
+# endif
+    if (dup2(fd, fd2) < 0)
+        return -1;
+    if (rpy_set_inheritable(fd2, 0) != 0) {
+        close(fd2);
+        return -1;
+    }
+    return 0;
+#endif
+}
+    """ % {'HAVE_DUP3': HAVE_DUP3}],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
 
 c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
@@ -2104,12 +2253,56 @@
 c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
                              compilation_info=eci_inheritable)
+c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                compilation_info=eci_inheritable)
+c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                 compilation_info=eci_inheritable)
 
 def set_inheritable(fd, inheritable):
-    error = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', error)
+    result = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', result)
 
 def get_inheritable(fd):
     res = c_get_inheritable(fd)
     res = handle_posix_error('get_inheritable', res)
     return res != 0
+
+class SetNonInheritableCache(object):
+    """Make one prebuilt instance of this for each path that creates
+    file descriptors, where you don't necessarily know if that function
+    returns inheritable or non-inheritable file descriptors.
+    """
+    _immutable_fields_ = ['cached_inheritable?']
+    cached_inheritable = -1    # -1 = don't know yet; 0 = off; 1 = on
+
+    def set_non_inheritable(self, fd):
+        if self.cached_inheritable == -1:
+            self.cached_inheritable = get_inheritable(fd)
+        if self.cached_inheritable == 1:
+            # 'fd' is inheritable; we must manually turn it off
+            set_inheritable(fd, False)
+
+    def _cleanup_(self):
+        self.cached_inheritable = -1
+
+class ENoSysCache(object):
+    """Cache whether a system call returns ENOSYS or not."""
+    _immutable_fields_ = ['cached_nosys?']
+    cached_nosys = -1      # -1 = don't know; 0 = no; 1 = yes, getting ENOSYS
+
+    def attempt_syscall(self):
+        return self.cached_nosys != 1
+
+    def fallback(self, res):
+        nosys = self.cached_nosys
+        if nosys == -1:
+            nosys = (res < 0 and get_saved_errno() == errno.ENOSYS)
+            self.cached_nosys = nosys
+        return nosys
+
+    def _cleanup_(self):
+        self.cached_nosys = -1
+
+_pipe2_syscall = ENoSysCache()
diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py
--- a/rpython/rlib/rsignal.py
+++ b/rpython/rlib/rsignal.py
@@ -31,7 +31,7 @@
     signal_names.append('CTRL_BREAK_EVENT')
     CTRL_C_EVENT = 0
     CTRL_BREAK_EVENT = 1
-includes = ['stdlib.h', 'src/signals.h']
+includes = ['stdlib.h', 'src/signals.h', 'signal.h']
 if sys.platform != 'win32':
     includes.append('sys/time.h')
 
@@ -47,7 +47,9 @@
     _compilation_info_ = eci
 
 if sys.platform != 'win32':
-    for name in """ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF""".split():
+    for name in """
+        ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF
+        SIG_BLOCK SIG_UNBLOCK SIG_SETMASK""".split():
         setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name))
 
     CConfig.timeval = rffi_platform.Struct(
@@ -71,7 +73,8 @@
 pypysig_default = external('pypysig_default', [rffi.INT], lltype.Void)
 pypysig_setflag = external('pypysig_setflag', [rffi.INT], lltype.Void)
 pypysig_reinstall = external('pypysig_reinstall', [rffi.INT], lltype.Void)
-pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd', [rffi.INT], rffi.INT)
+pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd',
+                                 [rffi.INT, rffi.INT], rffi.INT)
 pypysig_poll = external('pypysig_poll', [], rffi.INT, releasegil=False)
 # don't bother releasing the GIL around a call to pypysig_poll: it's
 # pointless and a performance issue
@@ -98,3 +101,20 @@
                            [rffi.INT, itimervalP, itimervalP], rffi.INT,
                            save_err=rffi.RFFI_SAVE_ERRNO)
     c_getitimer = external('getitimer', [rffi.INT, itimervalP], rffi.INT)
+
+c_pthread_kill = external('pthread_kill', [lltype.Signed, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+
+if sys.platform != 'win32':
+    c_sigset_t = rffi.COpaquePtr('sigset_t', compilation_info=eci)
+    c_sigemptyset = external('sigemptyset', [c_sigset_t], rffi.INT)
+    c_sigaddset = external('sigaddset', [c_sigset_t, rffi.INT], rffi.INT)
+    c_sigismember = external('sigismember', [c_sigset_t, rffi.INT], rffi.INT)
+    c_sigwait = external('sigwait', [c_sigset_t, rffi.INTP], rffi.INT,
+                         releasegil=True,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    c_sigpending = external('sigpending', [c_sigset_t], rffi.INT,
+                            save_err=rffi.RFFI_SAVE_ERRNO)
+    c_pthread_sigmask = external('pthread_sigmask',
+                                 [rffi.INT, c_sigset_t, c_sigset_t], rffi.INT,
+                                 save_err=rffi.RFFI_SAVE_ERRNO)
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -8,10 +8,11 @@
 # XXX this does not support yet the least common AF_xxx address families
 # supported by CPython.  See http://bugs.pypy.org/issue1942
 
+from errno import EINVAL
 from rpython.rlib import _rsocket_rffi as _c, jit, rgc
 from rpython.rlib.objectmodel import instantiate, keepalive_until_here
 from rpython.rlib.rarithmetic import intmask, r_uint
-from rpython.rlib import rthread
+from rpython.rlib import rthread, rposix
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rffi import sizeof, offsetof
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -522,12 +523,28 @@
     timeout = -1.0
 
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0,
-                 fd=_c.INVALID_SOCKET):
+                 fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            fd = _c.socket(family, type, proto)
-        if _c.invalid_socket(fd):
-            raise self.error_handler()
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socket() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                fd = _c.socket(family, type | SOCK_CLOEXEC, proto)
+                if fd < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise self.error_handler()
+            if _c.invalid_socket(fd):
+                fd = _c.socket(family, type, proto)
+                if _c.invalid_socket(fd):
+                    raise self.error_handler()
+                if not inheritable:
+                    sock_set_inheritable(fd, False)
         # PLAT RISCOS
         self.fd = fd
         self.family = family
@@ -630,20 +647,33 @@
         return addr, addr.addr_p, addrlen_p
 
     @jit.dont_look_inside
-    def accept(self):
+    def accept(self, inheritable=True):
         """Wait for an incoming connection.
         Return (new socket fd, client address)."""
         if self._select(False) == 1:
             raise SocketTimeout
         address, addr_p, addrlen_p = self._addrbuf()
         try:
-            newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+            remove_inheritable = not inheritable
+            if (not inheritable and SOCK_CLOEXEC is not None
+                    and _c.HAVE_ACCEPT4
+                    and _accept4_syscall.attempt_syscall()):
+                newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
+                                         SOCK_CLOEXEC)
+                if _accept4_syscall.fallback(newfd):
+                    newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+                else:
+                    remove_inheritable = False
+            else:
+                newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
             addrlen = addrlen_p[0]
         finally:
             lltype.free(addrlen_p, flavor='raw')
             address.unlock()
         if _c.invalid_socket(newfd):
             raise self.error_handler()
+        if remove_inheritable:
+            sock_set_inheritable(newfd, False)
         address.addrlen = rffi.cast(lltype.Signed, addrlen)
         return (newfd, address)
 
@@ -1032,6 +1062,12 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
+def sock_set_inheritable(fd, inheritable):
+    try:
+        rposix.set_inheritable(fd, inheritable)
+    except OSError as e:
+        raise CSocketError(e.errno)
+
 class SocketError(Exception):
     applevelerrcls = 'error'
     def __init__(self):
@@ -1090,7 +1126,7 @@
 
 if hasattr(_c, 'socketpair'):
     def socketpair(family=socketpair_default_family, type=SOCK_STREAM, proto=0,
-                   SocketClass=RSocket):
+                   SocketClass=RSocket, inheritable=True):
         """socketpair([family[, type[, proto]]]) -> (socket object, socket object)
 
         Create a pair of socket objects from the sockets returned by the platform
@@ -1100,18 +1136,41 @@
         """
         result = lltype.malloc(_c.socketpair_t, 2, flavor='raw')
         try:
-            res = _c.socketpair(family, type, proto, result)
+            res = -1
+            remove_inheritable = not inheritable
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socketpair() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                res = _c.socketpair(family, type | SOCK_CLOEXEC,
+                                    proto, result)
+                if res < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise last_error()
+                else:
+                    remove_inheritable = False
+            #
             if res < 0:
-                raise last_error()
+                res = _c.socketpair(family, type, proto, result)
+                if res < 0:
+                    raise last_error()
             fd0 = rffi.cast(lltype.Signed, result[0])
             fd1 = rffi.cast(lltype.Signed, result[1])
         finally:
             lltype.free(result, flavor='raw')
+        if remove_inheritable:
+            sock_set_inheritable(fd0, False)
+            sock_set_inheritable(fd1, False)
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
 if _c.WIN32:
-    def dup(fd):
+    def dup(fd, inheritable=True):
         with lltype.scoped_alloc(_c.WSAPROTOCOL_INFO, zero=True) as info:
             if _c.WSADuplicateSocket(fd, rwin32.GetCurrentProcessId(), info):
                 raise last_error()
@@ -1122,15 +1181,16 @@
                 raise last_error()
             return result
 else:
-    def dup(fd):
-        return _c.dup(fd)
-
-    def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
-        # Dup the fd so it and the socket can be closed independently
-        fd = _c.dup(fd)
+    def dup(fd, inheritable=True):
+        fd = rposix._dup(fd, inheritable)
         if fd < 0:
             raise last_error()
-        return make_socket(fd, family, type, proto, SocketClass)
+        return fd
+
+def fromfd(fd, family, type, proto=0, SocketClass=RSocket, inheritable=True):
+    # Dup the fd so it and the socket can be closed independently
+    fd = dup(fd, inheritable=inheritable)
+    return make_socket(fd, family, type, proto, SocketClass)
 
 def getdefaulttimeout():
     return defaults.timeout
@@ -1407,3 +1467,5 @@
     if timeout < 0.0:
         timeout = -1.0
     defaults.timeout = timeout
+
+_accept4_syscall = rposix.ENoSysCache()
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -24,12 +24,15 @@
 #### Constants
 
 # Identifying as _sre from Python 2.3 and onwards (at least up to 2.7)
-MAGIC = 20031017
+# UPDATE: change was necessary for Python 3.3 changes
+MAGIC = 20140917
 
 if sys.maxint > 2**32:
     MAXREPEAT = int(2**32 - 1)
+    MAXGROUPS = int(2**31 - 1)
 else:
     MAXREPEAT = int(2**31 - 1)
+    MAXGROUPS = int((2**31 / sys.maxint / 2) - 1)
 
 # In _sre.c this is bytesize of the code word type of the C implementation.
 # There it's 2 for normal Python builds and more for wide unicode builds (large 
diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -66,11 +66,13 @@
                'i': 'signed int',
                'l': 'signed long',
                'q': 'signed long long',
+               'n': 'ssize_t',
                'B': 'unsigned char',
                'H': 'unsigned short',
                'I': 'unsigned int',
                'L': 'unsigned long',
                'Q': 'unsigned long long',
+               'N': 'size_t',
                'P': 'char *',
                'f': 'float',
                'd': 'double',
@@ -78,8 +80,11 @@
                }
 
     pre_include_bits = ["""
+        #include <sys/types.h>
         #ifdef _MSC_VER
         #define _Bool char
+        typedef int ssize_t; /* XXX fixme for 64 bit*/
+        typedef unsigned int size_t; /* XXX fixme for 64 bit*/
         #endif"""]
     field_names = dict.fromkeys(INSPECT)
     for fmtchar, ctype in INSPECT.iteritems():
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -22,6 +22,16 @@
     include_dirs = [translator_c_dir],
 )
 
+class CConfig:
+    _compilation_info_ = eci
+    RPYTHREAD_NAME = rffi_platform.DefinedConstantString('RPYTHREAD_NAME')
+    USE_SEMAPHORES = rffi_platform.Defined('USE_SEMAPHORES')
+    CS_GNU_LIBPTHREAD_VERSION = rffi_platform.DefinedConstantInteger(
+        '_CS_GNU_LIBPTHREAD_VERSION')
+cconfig = rffi_platform.configure(CConfig)
+globals().update(cconfig)
+
+
 def llexternal(name, args, result, **kwds):
     kwds.setdefault('sandboxsafe', True)
     return rffi.llexternal(name, args, result, compilation_info=eci,
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1,7 +1,7 @@
 import sys
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
-from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import jit
@@ -74,7 +74,7 @@
 else:
     def code_to_unichr(code):
         # generate surrogates for large codes
-        return unichr_returns_surrogate(code)
+        return unichr_returns_surrogate(widen(code))
 
 def _STORECHAR(result, CH, byteorder):
     hi = chr(((CH) >> 8) & 0xff)
@@ -1381,7 +1381,7 @@
                 result.append(STR('\\\\'))
 
             # Map non-printable or non-ascii to '\xhh' or '\uhhhh'
-            elif pass_printable and not unicodedb.isprintable(oc):
+            elif pass_printable and not (oc <= 0x10ffff and unicodedb.isprintable(oc)):
                 char_escape_helper(result, oc)
             elif not pass_printable and (oc < 32 or oc >= 0x7F):
                 char_escape_helper(result, oc)
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -46,6 +46,7 @@
         LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP)
         LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP)
         LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.UINTP)
+        LPBOOL = rffi_platform.SimpleType("LPBOOL", rffi.LONGP)
         SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T)
         ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG)
 
@@ -58,6 +59,24 @@
         SYSTEMTIME = rffi_platform.Struct('SYSTEMTIME',
                                           [])
 
+        Struct = rffi_platform.Struct
+        COORD = Struct("COORD",
+                       [("X", rffi.SHORT),
+                        ("Y", rffi.SHORT)])
+
+        SMALL_RECT = Struct("SMALL_RECT",
+                            [("Left", rffi.SHORT),
+                             ("Top", rffi.SHORT),
+                             ("Right", rffi.SHORT),
+                             ("Bottom", rffi.SHORT)])
+
+        CONSOLE_SCREEN_BUFFER_INFO = Struct("CONSOLE_SCREEN_BUFFER_INFO",
+                                            [("dwSize", COORD),
+                                             ("dwCursorPosition", COORD),
+                                             ("wAttributes", WORD.ctype_hint),
+                                             ("srWindow", SMALL_RECT),
+                                             ("dwMaximumWindowSize", COORD)])
+
         OSVERSIONINFOEX = rffi_platform.Struct(
             'OSVERSIONINFOEX',
             [('dwOSVersionInfoSize', rffi.UINT),
@@ -92,7 +111,8 @@
                        PROCESS_VM_WRITE
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
-                       WC_NO_BEST_FIT_CHARS
+                       WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
+                       STD_ERROR_HANDLE
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -443,3 +463,13 @@
         return rffi.cast(lltype.Signed, _GetConsoleOutputCP())
 
     _wenviron_items, _wgetenv, _wputenv = make_env_impls(win32=True)
+
+
+    _GetStdHandle = winexternal(
+        'GetStdHandle', [DWORD], HANDLE)
+
+    def GetStdHandle(handle_id):
+        return _GetStdHandle(handle_id)
+    CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
+    GetConsoleScreenBufferInfo = winexternal(
+        "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -324,7 +324,8 @@
     return data
 
 
-def decompress(stream, data, flush=Z_SYNC_FLUSH, max_length=sys.maxint):
+def decompress(stream, data, flush=Z_SYNC_FLUSH, max_length=sys.maxint,
+               zdict=None):
     """
     Feed more data into an inflate stream.  Returns a tuple (string,
     finished, unused_data_length).  The string contains (a part of) the
@@ -350,7 +351,7 @@
         should_finish = False
     while_doing = "while decompressing data"
     data, err, avail_in = _operate(stream, data, flush, max_length, _inflate,
-                                   while_doing)
+                                   while_doing, zdict=zdict)
     if should_finish:
         # detect incomplete input
         rffi.setintfield(stream, 'c_avail_in', 0)
@@ -361,7 +362,7 @@
     return data, finished, avail_in
 
 
-def _operate(stream, data, flush, max_length, cfunc, while_doing):
+def _operate(stream, data, flush, max_length, cfunc, while_doing, zdict=None):
     """Common code for compress() and decompress().
     """
     # Prepare the input buffer for the stream
@@ -388,6 +389,10 @@
                 max_length -= bufsize
                 rffi.setintfield(stream, 'c_avail_out', bufsize)
                 err = cfunc(stream, flush)
+                if err == Z_NEED_DICT and zdict is not None:
+                    inflateSetDictionary(stream, zdict)
+                    # repeat the call to inflate
+                    err = cfunc(stream, flush)
                 if err == Z_OK or err == Z_STREAM_END:
                     # accumulate data into 'result'
                     avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py
--- a/rpython/rlib/test/test_rbigint.py
+++ b/rpython/rlib/test/test_rbigint.py
@@ -674,6 +674,11 @@
                 else:
                     assert ulps_check(l, math.log(op)) is None
 
+    def test_log2(self):
+        assert rbigint.fromlong(1).log(2.0) == 0.0
+        assert rbigint.fromlong(2).log(2.0) == 1.0
+        assert rbigint.fromlong(2**1023).log(2.0) == 1023.0
+
 class TestInternalFunctions(object):
     def test__inplace_divrem1(self):
         # signs are not handled in the helpers!
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -589,3 +589,18 @@
     assert rposix.get_inheritable(fd1) == False
     os.close(fd1)
     os.close(fd2)
+
+def test_SetNonInheritableCache():
+    cache = rposix.SetNonInheritableCache()
+    fd1, fd2 = os.pipe()
+    assert rposix.get_inheritable(fd1) == True
+    assert rposix.get_inheritable(fd1) == True
+    assert cache.cached_inheritable == -1
+    cache.set_non_inheritable(fd1)
+    assert cache.cached_inheritable == 1
+    cache.set_non_inheritable(fd2)
+    assert cache.cached_inheritable == 1
+    assert rposix.get_inheritable(fd1) == False
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -119,6 +119,16 @@
     s1.close()
     s2.close()
 
+def test_socketpair_inheritable():
+    if sys.platform == "win32":
+        py.test.skip('No socketpair on Windows')
+    for inh in [False, True]:
+        s1, s2 = socketpair(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        assert rposix.get_inheritable(s2.fd) == inh
+        s1.close()
+        s2.close()
+
 def test_socketpair_recvinto_1():
     class Buffer:
         def setslice(self, start, string):
@@ -378,6 +388,12 @@
     s1.close()
     s2.close()
 
+def test_inheritable():
+    for inh in [False, True]:
+        s1 = RSocket(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        s1.close()
+
 def test_getaddrinfo_http():
     lst = getaddrinfo('localhost', 'http')
     assert isinstance(lst, list)
diff --git a/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt b/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt
@@ -0,0 +1,912 @@
+# CaseFolding-3.2.0.txt
+# Date: 2002-03-22,20:54:33 GMT [MD]
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#65533;e" to match.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, see
+# UTR #21 Case Mappings, at http://www.unicode.org/unicode/reports/tr21/
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F2; C; 03C3; # GREEK LUNATE SIGMA SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N
+24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O
+24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P
+24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q
+24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R
+24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S
+24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T
+24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U
+24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V
+24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W
+24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X
+24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y
+24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z
+FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF
+FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI
+FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
+FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI
+FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL
+FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T
+FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST
+FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW
+FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH
+FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI
+FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW
+FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH
+FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A
+FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B
+FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C
+FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D
+FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E
+FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F
+FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G
+FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H
+FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I
+FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J
+FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K
+FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L
+FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M
+FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N
+FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O
+FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P
+FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q
+FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R
+FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S
+FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T
+FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U
+FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V
+FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W
+FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X
+FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y
+FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
+10400; C; 10428; # DESERET CAPITAL LETTER LONG I
+10401; C; 10429; # DESERET CAPITAL LETTER LONG E
+10402; C; 1042A; # DESERET CAPITAL LETTER LONG A
+10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH
+10404; C; 1042C; # DESERET CAPITAL LETTER LONG O
+10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO
+10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I
+10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E
+10408; C; 10430; # DESERET CAPITAL LETTER SHORT A
+10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH
+1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O
+1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO
+1040C; C; 10434; # DESERET CAPITAL LETTER AY
+1040D; C; 10435; # DESERET CAPITAL LETTER OW
+1040E; C; 10436; # DESERET CAPITAL LETTER WU
+1040F; C; 10437; # DESERET CAPITAL LETTER YEE
+10410; C; 10438; # DESERET CAPITAL LETTER H
+10411; C; 10439; # DESERET CAPITAL LETTER PEE
+10412; C; 1043A; # DESERET CAPITAL LETTER BEE
+10413; C; 1043B; # DESERET CAPITAL LETTER TEE
+10414; C; 1043C; # DESERET CAPITAL LETTER DEE
+10415; C; 1043D; # DESERET CAPITAL LETTER CHEE
+10416; C; 1043E; # DESERET CAPITAL LETTER JEE
+10417; C; 1043F; # DESERET CAPITAL LETTER KAY
+10418; C; 10440; # DESERET CAPITAL LETTER GAY
+10419; C; 10441; # DESERET CAPITAL LETTER EF
+1041A; C; 10442; # DESERET CAPITAL LETTER VEE
+1041B; C; 10443; # DESERET CAPITAL LETTER ETH
+1041C; C; 10444; # DESERET CAPITAL LETTER THEE
+1041D; C; 10445; # DESERET CAPITAL LETTER ES
+1041E; C; 10446; # DESERET CAPITAL LETTER ZEE
+1041F; C; 10447; # DESERET CAPITAL LETTER ESH
+10420; C; 10448; # DESERET CAPITAL LETTER ZHEE
+10421; C; 10449; # DESERET CAPITAL LETTER ER
+10422; C; 1044A; # DESERET CAPITAL LETTER EL
+10423; C; 1044B; # DESERET CAPITAL LETTER EM
+10424; C; 1044C; # DESERET CAPITAL LETTER EN
+10425; C; 1044D; # DESERET CAPITAL LETTER ENG
diff --git a/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt b/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt
@@ -0,0 +1,1202 @@
+# CaseFolding-5.2.0.txt
+# Date: 2009-05-28, 23:02:34 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2009 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#223;e" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding 
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard, Version 5.0.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+# @missing 0000..10FFFF; <codepoint>
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE

From pypy.commits at gmail.com  Sun Aug 28 01:19:11 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 Aug 2016 22:19:11 -0700 (PDT)
Subject: [pypy-commit] pypy rpython-resync: Kill the rsre_char.MAGIC nonsense
Message-ID: <57c2744f.469d1c0a.bcbc7.489c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rpython-resync
Changeset: r86632:f48df4d5e6d9
Date: 2016-08-28 06:13 +0100
http://bitbucket.org/pypy/pypy/changeset/f48df4d5e6d9/

Log:	Kill the rsre_char.MAGIC nonsense

diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20031017)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -23,10 +23,6 @@
 
 #### Constants
 
-# Identifying as _sre from Python 2.3 and onwards (at least up to 2.7)
-# UPDATE: change was necessary for Python 3.3 changes
-MAGIC = 20140917
-
 if sys.maxint > 2**32:
     MAXREPEAT = int(2**32 - 1)
     MAXGROUPS = int(2**31 - 1)
@@ -35,7 +31,7 @@
     MAXGROUPS = int((2**31 / sys.maxint / 2) - 1)
 
 # In _sre.c this is bytesize of the code word type of the C implementation.
-# There it's 2 for normal Python builds and more for wide unicode builds (large 
+# There it's 2 for normal Python builds and more for wide unicode builds (large
 # enough to hold a 32-bit UCS-4 encoded character). Since here in pure Python
 # we only see re bytecodes as Python longs, we shouldn't have to care about the
 # codesize. But sre_compile will compile some stuff differently depending on the

From pypy.commits at gmail.com  Sun Aug 28 01:27:31 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 27 Aug 2016 22:27:31 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: correct divergence-from-release hash
Message-ID: <57c27643.05371c0a.8370b.408d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86633:b55cdc13dc51
Date: 2016-08-28 11:06 +1000
http://bitbucket.org/pypy/pypy/changeset/b55cdc13dc51/

Log:	correct divergence-from-release hash

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,6 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 4176c6f63109
+.. startrev: 522736f816dc
 
 

From pypy.commits at gmail.com  Sun Aug 28 01:30:03 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 27 Aug 2016 22:30:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: update repackage script
Message-ID: <57c276db.8aacc20a.f3495.026b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86634:68bb3510d821
Date: 2016-08-28 15:28 +1000
http://bitbucket.org/pypy/pypy/changeset/68bb3510d821/

Log:	update repackage script

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=3
-rev=1
+min=4
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Sun Aug 28 01:30:05 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 27 Aug 2016 22:30:05 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: update repackage script
Message-ID: <57c276dd.274fc20a.2eca5.104e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86635:77392ad26350
Date: 2016-08-28 15:28 +1000
http://bitbucket.org/pypy/pypy/changeset/77392ad26350/

Log:	update repackage script

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=3
-rev=1
+min=4
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Sun Aug 28 05:55:32 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sun, 28 Aug 2016 02:55:32 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase
Message-ID: <57c2b514.e2efc20a.ff32c.4992@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86636:041e0b606934
Date: 2016-08-27 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/041e0b606934/

Log:	Update qcgc codebase

diff --git a/rpython/translator/c/src/qcgc/allocator.c b/rpython/translator/c/src/qcgc/allocator.c
--- a/rpython/translator/c/src/qcgc/allocator.c
+++ b/rpython/translator/c/src/qcgc/allocator.c
@@ -2,12 +2,14 @@
 
 #include <assert.h>
 #include <stdbool.h>
+#include <stdlib.h>
 #include <string.h>
 
+#include "hugeblocktable.h"
+
 QCGC_STATIC size_t bytes_to_cells(size_t bytes);
 
 QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells);
-QCGC_STATIC cell_t *bump_allocator_allocate(size_t cells);
 QCGC_STATIC void bump_allocator_advance(size_t cells);
 
 QCGC_STATIC bool is_small(size_t cells);
@@ -15,7 +17,6 @@
 QCGC_STATIC size_t large_index(size_t cells);
 QCGC_STATIC size_t small_index_to_cells(size_t index);
 
-QCGC_STATIC cell_t *fit_allocator_allocate(size_t cells);
 QCGC_STATIC cell_t *fit_allocator_small_first_fit(size_t index, size_t cells);
 QCGC_STATIC cell_t *fit_allocator_large_fit(size_t index, size_t cells);
 QCGC_STATIC cell_t *fit_allocator_large_first_fit(size_t index, size_t cells);
@@ -61,28 +62,6 @@
 	free(qcgc_allocator_state.arenas);
 }
 
-cell_t *qcgc_allocator_allocate(size_t bytes) {
-	size_t size_in_cells = bytes_to_cells(bytes);
-#if CHECKED
-	assert(size_in_cells > 0);
-	assert(size_in_cells <= QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
-#endif
-	cell_t *result;
-
-	// TODO: Implement switch for bump/fit allocator
-	if (true) {
-		result = bump_allocator_allocate(size_in_cells);
-	} else {
-		result = fit_allocator_allocate(size_in_cells);
-	}
-
-	qcgc_arena_mark_allocated(result, size_in_cells);
-#if QCGC_INIT_ZERO
-	memset(result, 0, bytes);
-#endif
-	return result;
-}
-
 void qcgc_fit_allocator_add(cell_t *ptr, size_t cells) {
 	if (cells > 0) {
 		if (is_small(cells)) {
@@ -111,36 +90,75 @@
 	qcgc_allocator_state.bump_state.remaining_cells -= cells;
 }
 
-QCGC_STATIC cell_t *bump_allocator_allocate(size_t cells) {
+/*******************************************************************************
+ * Allocators                                                                  *
+ ******************************************************************************/
+
+object_t *qcgc_bump_allocate(size_t bytes) {
+	size_t cells = bytes_to_cells(bytes);
 	if (cells > qcgc_allocator_state.bump_state.remaining_cells) {
 		// Grab a new arena
+		// FIXME: Add remaining memory to fit allocator
 		arena_t *arena = qcgc_arena_create();
 		bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
 				QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
 		qcgc_allocator_state.arenas =
 			qcgc_arena_bag_add(qcgc_allocator_state.arenas, arena);
 	}
-	cell_t *result = qcgc_allocator_state.bump_state.bump_ptr;
+	cell_t *mem = qcgc_allocator_state.bump_state.bump_ptr;
 	bump_allocator_advance(cells);
+
+	qcgc_arena_mark_allocated(mem, cells);
+	object_t *result = (object_t *) mem;
+
+#if QCGC_INIT_ZERO
+	memset(result, 0, cells * sizeof(cell_t));
+#endif
+
+	result->flags |= QCGC_GRAY_FLAG;
 	return result;
 }
 
-QCGC_STATIC cell_t *fit_allocator_allocate(size_t cells) {
-	cell_t *result;
+object_t *qcgc_fit_allocate(size_t bytes) {
+	size_t cells = bytes_to_cells(bytes);
+	cell_t *mem;
 
 	if (is_small(cells)) {
 		size_t index = small_index(cells);
-		result = fit_allocator_small_first_fit(index, cells);
+		mem = fit_allocator_small_first_fit(index, cells);
 	} else {
 		size_t index = large_index(cells);
-		result = fit_allocator_large_fit(index, cells);
+		mem = fit_allocator_large_fit(index, cells);
 	}
 
-	if (result == NULL) {
-		// No valid block found
-		result = bump_allocator_allocate(cells);
+	if (mem == NULL) {
+		return NULL;
 	}
 
+	qcgc_arena_mark_allocated(mem, cells);
+	object_t *result = (object_t *) mem;
+
+#if QCGC_INIT_ZERO
+	memset(result, 0, cells * sizeof(cell_t));
+#endif
+
+	result->flags |= QCGC_GRAY_FLAG;
+	return result;
+}
+
+/**
+ * Constraints:
+ * - Zero initialized
+ * - Aligned to arena size
+ * - Multiple of arena size
+ * - No header, metadata stored in hash-map
+ */
+object_t *qcgc_large_allocate(size_t bytes) {
+	object_t *result = aligned_alloc(QCGC_ARENA_SIZE, bytes);
+#if QCGC_INIT_ZERO
+	memset(result, 0, bytes);
+#endif
+	qcgc_hbtable_insert(result);
 	return result;
 }
 
diff --git a/rpython/translator/c/src/qcgc/allocator.h b/rpython/translator/c/src/qcgc/allocator.h
--- a/rpython/translator/c/src/qcgc/allocator.h
+++ b/rpython/translator/c/src/qcgc/allocator.h
@@ -57,13 +57,32 @@
 void qcgc_allocator_destroy(void);
 
 /**
- * Allocate new memory region
+ * Allocate new memory region using fit allocator
+ *
+ * @param	bytes	Desired size of the memory region in bytes
+ * @return	Pointer to memory large enough to hold size bytes, NULL in case of
+ *			errors or if there is no block sufficently large block, already zero
+ *			initialized if QCGC_INIT_ZERO is set
+ */
+object_t *qcgc_fit_allocate(size_t bytes);
+
+/**
+ * Allocate new memory region using bump allocator
  *
  * @param	bytes	Desired size of the memory region in bytes
  * @return	Pointer to memory large enough to hold size bytes, NULL in case of
  *			errors, already zero initialized if QCGC_INIT_ZERO is set
  */
-cell_t *qcgc_allocator_allocate(size_t bytes);
+object_t *qcgc_bump_allocate(size_t bytes);
+
+/**
+ * Allocate new memory region using huge block allocator
+ *
+ * @param	bytes	Desired size of the memory region in bytes
+ * @return	Pointer to memory large enough to hold size bytes, NULL in case of
+ *			errors, already zero initialized if QCGC_INIT_ZERO is set
+ */
+object_t *qcgc_large_allocate(size_t bytes);
 
 
 /**
diff --git a/rpython/translator/c/src/qcgc/bag.c b/rpython/translator/c/src/qcgc/bag.c
--- a/rpython/translator/c/src/qcgc/bag.c
+++ b/rpython/translator/c/src/qcgc/bag.c
@@ -5,3 +5,4 @@
 DEFINE_BAG(arena_bag, arena_t *);
 DEFINE_BAG(linear_free_list, cell_t *);
 DEFINE_BAG(exp_free_list, struct exp_free_list_item_s);
+DEFINE_BAG(hbbucket, struct hbtable_entry_s);
diff --git a/rpython/translator/c/src/qcgc/bag.h b/rpython/translator/c/src/qcgc/bag.h
--- a/rpython/translator/c/src/qcgc/bag.h
+++ b/rpython/translator/c/src/qcgc/bag.h
@@ -14,13 +14,23 @@
 	type items[];															\
 } name##_t;																	\
 																			\
+__attribute__ ((warn_unused_result))										\
 name##_t *qcgc_##name##_create(size_t size);								\
+																			\
+__attribute__ ((warn_unused_result))										\
 name##_t *qcgc_##name##_add(name##_t *self, type item);						\
+																			\
+__attribute__ ((warn_unused_result))										\
 name##_t *qcgc_##name##_remove_index(name##_t *self, size_t index);
 
 #define DEFINE_BAG(name, type)												\
+																			\
 QCGC_STATIC size_t name##_size(size_t size);								\
+																			\
+__attribute__ ((warn_unused_result))										\
 QCGC_STATIC name##_t *name##_grow(name##_t *self);							\
+																			\
+__attribute__ ((warn_unused_result))										\
 QCGC_STATIC name##_t *name##_shrink(name##_t *self);						\
 																			\
 name##_t *qcgc_##name##_create(size_t size) {								\
@@ -77,6 +87,12 @@
 	size_t size;
 };
 
+struct hbtable_entry_s {
+	object_t *object;
+	bool mark_flag;
+};
+
 DECLARE_BAG(arena_bag, arena_t *);
 DECLARE_BAG(linear_free_list, cell_t *);
 DECLARE_BAG(exp_free_list, struct exp_free_list_item_s);
+DECLARE_BAG(hbbucket, struct hbtable_entry_s);
diff --git a/rpython/translator/c/src/qcgc/gc_state.h b/rpython/translator/c/src/qcgc/gc_state.h
--- a/rpython/translator/c/src/qcgc/gc_state.h
+++ b/rpython/translator/c/src/qcgc/gc_state.h
@@ -25,6 +25,7 @@
 struct qcgc_state {
 	shadow_stack_t *shadow_stack;
 	shadow_stack_t *prebuilt_objects;
+	gray_stack_t *gp_gray_stack;
 	size_t gray_stack_size;
 	gc_phase_t phase;
 } qcgc_state;
diff --git a/rpython/translator/c/src/qcgc/gray_stack.h b/rpython/translator/c/src/qcgc/gray_stack.h
--- a/rpython/translator/c/src/qcgc/gray_stack.h
+++ b/rpython/translator/c/src/qcgc/gray_stack.h
@@ -12,8 +12,13 @@
 	object_t *items[];
 } gray_stack_t;
 
+__attribute__ ((warn_unused_result))
 gray_stack_t *qcgc_gray_stack_create(size_t size);
 
+__attribute__ ((warn_unused_result))
 gray_stack_t *qcgc_gray_stack_push(gray_stack_t *stack, object_t *item);
+
 object_t *qcgc_gray_stack_top(gray_stack_t *stack);
+
+__attribute__ ((warn_unused_result))
 gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack);
diff --git a/rpython/translator/c/src/qcgc/hugeblocktable.c b/rpython/translator/c/src/qcgc/hugeblocktable.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/hugeblocktable.c
@@ -0,0 +1,80 @@
+#include "hugeblocktable.h"
+
+#include <assert.h>
+
+#include "gc_state.h"
+
+QCGC_STATIC size_t bucket(object_t *object);
+
+void qcgc_hbtable_initialize(void) {
+	qcgc_hbtable.mark_flag_ref = false;
+	for (size_t i = 0; i < QCGC_HBTABLE_BUCKETS; i++) {
+		qcgc_hbtable.bucket[i] = qcgc_hbbucket_create(4);
+	}
+}
+
+void qcgc_hbtable_destroy(void) {
+	for (size_t i = 0; i < QCGC_HBTABLE_BUCKETS; i++) {
+		free(qcgc_hbtable.bucket[i]);
+	}
+}
+
+void qcgc_hbtable_insert(object_t *object) {
+	size_t i = bucket(object);
+	qcgc_hbtable.bucket[i] = qcgc_hbbucket_add(qcgc_hbtable.bucket[i],
+			(struct hbtable_entry_s) {
+				.object = object,
+				.mark_flag = !qcgc_hbtable.mark_flag_ref});
+}
+
+bool qcgc_hbtable_mark(object_t *object) {
+	hbbucket_t *b = qcgc_hbtable.bucket[bucket(object)];
+	size_t count = b->count;
+	for (size_t i = 0; i < count; i++) {
+		if (b->items[i].object == object) {
+			if (b->items[i].mark_flag != qcgc_hbtable.mark_flag_ref) {
+				b->items[i].mark_flag = qcgc_hbtable.mark_flag_ref;
+				return true;
+			}
+			return false;
+		}
+	}
+#if CHECKED
+	assert(false);
+#endif
+	return false;
+}
+
+bool qcgc_hbtable_is_marked(object_t *object) {
+	hbbucket_t *b = qcgc_hbtable.bucket[bucket(object)];
+	size_t count = b->count;
+	for (size_t i = 0; i < count; i++) {
+		if (b->items[i].object == object) {
+			return b->items[i].mark_flag == qcgc_hbtable.mark_flag_ref;
+		}
+	}
+	return false;
+}
+
+void qcgc_hbtable_sweep(void) {
+	for (size_t i = 0; i < QCGC_HBTABLE_BUCKETS; i++) {
+		hbbucket_t *b = qcgc_hbtable.bucket[i];
+		size_t j = 0;
+		while(j < b->count) {
+			if (b->items[j].mark_flag != qcgc_hbtable.mark_flag_ref) {
+				// White object
+				free(b->items[j].object);
+				b = qcgc_hbbucket_remove_index(b, j);
+			} else {
+				// Black object
+				j++;
+			}
+		}
+		qcgc_hbtable.bucket[i] = b;
+	}
+	qcgc_hbtable.mark_flag_ref = !qcgc_hbtable.mark_flag_ref;
+}
+
+QCGC_STATIC size_t bucket(object_t *object) {
+	return ((uintptr_t) object >> (QCGC_ARENA_SIZE_EXP)) % QCGC_HBTABLE_BUCKETS;
+}
diff --git a/rpython/translator/c/src/qcgc/hugeblocktable.h b/rpython/translator/c/src/qcgc/hugeblocktable.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/hugeblocktable.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "config.h"
+
+#include <stdbool.h>
+
+#include "bag.h"
+#include "object.h"
+#include "gray_stack.h"
+
+// Choosing a prime number, hoping for good results
+#define QCGC_HBTABLE_BUCKETS 61
+
+struct hbtable_s {
+	bool mark_flag_ref;
+	hbbucket_t *bucket[QCGC_HBTABLE_BUCKETS];
+} qcgc_hbtable;
+
+void qcgc_hbtable_initialize(void);
+void qcgc_hbtable_destroy(void);
+void qcgc_hbtable_insert(object_t *object);
+bool qcgc_hbtable_mark(object_t *object);
+bool qcgc_hbtable_is_marked(object_t *object);
+void qcgc_hbtable_sweep(void);
diff --git a/rpython/translator/c/src/qcgc/mark_list.c b/rpython/translator/c/src/qcgc/mark_list.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/mark_list.c
+++ /dev/null
@@ -1,180 +0,0 @@
-#include "mark_list.h"
-
-#include <assert.h>
-
-#include <stdlib.h>
-#include <string.h>
-
-QCGC_STATIC mark_list_t *qcgc_mark_list_grow(mark_list_t *list);
-QCGC_STATIC void qcgc_mark_list_check_invariant(mark_list_t *list);
-
-mark_list_t *qcgc_mark_list_create(size_t initial_size) {
-	size_t length = (initial_size + QCGC_MARK_LIST_SEGMENT_SIZE - 1) / QCGC_MARK_LIST_SEGMENT_SIZE;
-	length += (size_t) length == 0;
-	mark_list_t *result = (mark_list_t *)
-		malloc(sizeof(mark_list_t) + length * sizeof(object_t **));
-	result->head = 0;
-	result->tail = 0;
-	result->length = length;
-	result->insert_index = 0;
-	result->count = 0;
-	result->segments[result->head] = (object_t **)
-		calloc(QCGC_MARK_LIST_SEGMENT_SIZE, sizeof(object_t *));
-#if CHECKED
-	qcgc_mark_list_check_invariant(result);
-#endif
-	return result;
-}
-
-void qcgc_mark_list_destroy(mark_list_t *list) {
-#if CHECKED
-	qcgc_mark_list_check_invariant(list);
-#endif
-
-	size_t i = list->head;
-	while (i != list->tail) {
-		free(list->segments[i]);
-		i = (i + 1) % list->length;
-	}
-	free(list->segments[list->tail]);
-	free(list);
-}
-
-mark_list_t *qcgc_mark_list_push(mark_list_t *list, object_t *object) {
-#if CHECKED
-	assert(list != NULL);
-	assert(object != NULL);
-
-	qcgc_mark_list_check_invariant(list);
-	size_t old_count = list->count;
-#endif
-	if (list->insert_index >= QCGC_MARK_LIST_SEGMENT_SIZE) {
-		if ((list->tail + 1) % list->length == list->head) {
-			list = qcgc_mark_list_grow(list);
-		}
-		list->insert_index = 0;
-		list->tail = (list->tail + 1) % list->length;
-		list->segments[list->tail] = (object_t **)
-			calloc(QCGC_MARK_LIST_SEGMENT_SIZE, sizeof(object_t *));
-	}
-	list->segments[list->tail][list->insert_index] = object;
-	list->insert_index++;
-	list->count++;
-#if CHECKED
-	assert(list->count == old_count + 1);
-	assert(list->segments[list->tail][list->insert_index - 1] == object);
-	qcgc_mark_list_check_invariant(list);
-#endif
-	return list;
-}
-
-mark_list_t *qcgc_mark_list_push_all(mark_list_t *list,
-		object_t **objects, size_t count) {
-#if CHECKED
-	assert(list != NULL);
-	assert(objects != NULL);
-
-	qcgc_mark_list_check_invariant(list);
-
-	size_t old_count = list->count;
-	for (size_t i = 0; i < count; i++) {
-		assert(objects[i] != NULL);
-	}
-#endif
-	// FIXME: Optimize or remove
-	for (size_t i = 0; i < count; i++) {
-		list = qcgc_mark_list_push(list, objects[i]);
-	}
-#if CHECKED
-	assert(list->count == old_count + count);
-	qcgc_mark_list_check_invariant(list);
-#endif
-	return list;
-}
-
-object_t **qcgc_mark_list_get_head_segment(mark_list_t *list) {
-#if CHECKED
-	assert(list != NULL);
-	assert(list->segments[list->head] != NULL);
-	qcgc_mark_list_check_invariant(list);
-#endif
-	return list->segments[list->head];
-}
-
-mark_list_t *qcgc_mark_list_drop_head_segment(mark_list_t *list) {
-#if CHECKED
-	assert(list != NULL);
-	size_t old_head = list->head;
-	size_t old_tail = list->tail;
-	qcgc_mark_list_check_invariant(list);
-#endif
-	if (list->head != list->tail) {
-		free(list->segments[list->head]);
-		list->segments[list->head] = NULL;
-		list->head = (list->head + 1) % list->length;
-		list->count -= QCGC_MARK_LIST_SEGMENT_SIZE;
-	} else {
-		memset(list->segments[list->head], 0,
-				sizeof(object_t *) * QCGC_MARK_LIST_SEGMENT_SIZE);
-		list->insert_index = 0;
-		list->count = 0;
-	}
-#if CHECKED
-	assert(old_tail == list->tail);
-	if (old_head == old_tail) {
-		assert(old_head == list->head);
-	} else {
-		assert((old_head + 1) % list->length == list->head);
-	}
-	qcgc_mark_list_check_invariant(list);
-#endif
-	return list;
-}
-
-QCGC_STATIC mark_list_t *qcgc_mark_list_grow(mark_list_t *list) {
-#if CHECKED
-	assert(list != NULL);
-	size_t old_length = list->length;
-	size_t old_tail = list->tail;
-	qcgc_mark_list_check_invariant(list);
-#endif
-	mark_list_t *new_list = (mark_list_t *) realloc(list,
-			sizeof(mark_list_t) + 2 * list->length * sizeof(object_t **));
-	if (new_list->tail < new_list->head) {
-		memcpy(new_list->segments + new_list->length,
-				new_list->segments, (new_list->tail + 1) * sizeof(object_t **));
-		new_list->tail = new_list->length + new_list->tail;
-	}
-	new_list->length = 2 * new_list->length;
-#if CHECKED
-	assert(new_list->length == 2 * old_length);
-	if (old_tail < new_list->head) {
-		assert(new_list->tail == old_tail + old_length);
-		for (size_t i = 0; i < old_tail; i++) {
-			assert(new_list->segments[i] == new_list->segments[i + old_length]);
-		}
-	} else {
-		assert(new_list->tail == old_tail);
-	}
-	qcgc_mark_list_check_invariant(new_list);
-#endif
-	return new_list;
-}
-
-QCGC_STATIC void qcgc_mark_list_check_invariant(mark_list_t *list) {
-	assert(list->head < list->length);
-	assert(list->tail < list->length);
-	assert(list->count == (list->tail - list->head + list->length) % list->length * QCGC_MARK_LIST_SEGMENT_SIZE + list->insert_index);
-	for (size_t i = 0; i < list->length; i++) {
-		if ((list->head <= i && i <= list->tail) || (list->tail < list->head &&
-				(i <= list->tail || i >= list->head))) {
-			for (size_t j = 0; j < QCGC_MARK_LIST_SEGMENT_SIZE; j++) {
-				if (i != list->tail || j < list->insert_index) {
-					assert(list->segments[i][j] != NULL);
-				} else {
-					assert(list->segments[i][j] == NULL);
-				}
-			}
-		}
-	}
-}
diff --git a/rpython/translator/c/src/qcgc/mark_list.h b/rpython/translator/c/src/qcgc/mark_list.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/mark_list.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * @file	mark_list.h
- *
- * Object list for marking step
- */
-
-#pragma once
-
-#include "config.h"
-
-#include <stddef.h>
-
-#include "object.h"
-
-/**
- * Mark list - circular buffer.
- */
-typedef struct mark_list_s {
-	size_t head;
-	size_t tail;
-	size_t length;
-	size_t count;
-	size_t insert_index;
-	object_t **segments[];
-} mark_list_t;
-
-mark_list_t *qcgc_mark_list_create(size_t initial_size);
-void qcgc_mark_list_destroy(mark_list_t *list);
-
-mark_list_t *qcgc_mark_list_push(mark_list_t *list, object_t *object);
-mark_list_t *qcgc_mark_list_push_all(mark_list_t *list,
-		object_t **objects, size_t count);
-
-object_t **qcgc_mark_list_get_head_segment(mark_list_t *list);
-mark_list_t *qcgc_mark_list_drop_head_segment(mark_list_t *list);
diff --git a/rpython/translator/c/src/qcgc/object.h b/rpython/translator/c/src/qcgc/object.h
--- a/rpython/translator/c/src/qcgc/object.h
+++ b/rpython/translator/c/src/qcgc/object.h
@@ -4,8 +4,9 @@
 #include "config.h"
 #include <stdint.h>
 
-#define QCGC_GRAY_FLAG 0x01
-#define QCGC_PREBUILT_OBJECT 0x02
+#define QCGC_GRAY_FLAG (1<<0)
+#define QCGC_PREBUILT_OBJECT (1<<1)
+#define QCGC_PREBUILT_REGISTERED (1<<2)
 
 typedef struct object_s {
 	uint32_t flags;
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -7,15 +7,14 @@
 #include <string.h>
 
 #include "allocator.h"
+#include "hugeblocktable.h"
 #include "event_logger.h"
 
 // TODO: Eventually move to own header?
 #define MAX(a,b) (((a)>(b))?(a):(b))
 #define MIN(a,b) (((a)<(b))?(a):(b))
 
-void qcgc_mark(void);
-void qcgc_mark_all(void);
-void qcgc_mark_incremental(void);
+void qcgc_mark(bool incremental);
 void qcgc_pop_object(object_t *object);
 void qcgc_push_object(object_t *object);
 void qcgc_sweep(void);
@@ -23,22 +22,34 @@
 void qcgc_initialize(void) {
 	qcgc_state.shadow_stack = qcgc_shadow_stack_create(QCGC_SHADOWSTACK_SIZE);
 	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); //XXX
+	qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
 	qcgc_state.gray_stack_size = 0;
 	qcgc_state.phase = GC_PAUSE;
 	qcgc_allocator_initialize();
+	qcgc_hbtable_initialize();
 	qcgc_event_logger_initialize();
 }
 
 void qcgc_destroy(void) {
 	qcgc_event_logger_destroy();
+	qcgc_hbtable_destroy();
 	qcgc_allocator_destroy();
 	free(qcgc_state.shadow_stack);
+	free(qcgc_state.prebuilt_objects);
+	free(qcgc_state.gp_gray_stack);
 }
 
 /**
  * Shadow stack
  */
 void qcgc_shadowstack_push(object_t *object) {
+#if CHECKED
+	assert((object->flags & QCGC_PREBUILT_OBJECT) == 0);
+#endif
+	if (qcgc_state.phase != GC_PAUSE) {
+		qcgc_state.phase = GC_MARK;
+		qcgc_push_object(object);
+	}
 	qcgc_state.shadow_stack =
 		qcgc_shadow_stack_push(qcgc_state.shadow_stack, object);
 }
@@ -56,18 +67,45 @@
 #if CHECKED
 	assert(object != NULL);
 #endif
-	if ((object->flags & QCGC_GRAY_FLAG) == 0) {
-		object->flags |= QCGC_GRAY_FLAG;
-		if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
-			// Save prebuilt object into list
-			qcgc_shadow_stack_push(qcgc_state.prebuilt_objects, object);
-		} else if (qcgc_state.phase != GC_PAUSE) {
-			if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK) {
-				// This was black before, push it to gray stack again
-				arena_t *arena = qcgc_arena_addr((cell_t *) object);
-				arena->gray_stack = qcgc_gray_stack_push(
-						arena->gray_stack, object);
-			}
+	if ((object->flags & QCGC_GRAY_FLAG) != 0) {
+		// Already gray, skip
+		return;
+	}
+	object->flags |= QCGC_GRAY_FLAG;
+
+	// Register prebuilt object if necessary
+	if (((object->flags & QCGC_PREBUILT_OBJECT) != 0) &&
+			((object->flags & QCGC_PREBUILT_REGISTERED) == 0)) {
+		object->flags |= QCGC_PREBUILT_REGISTERED;
+		qcgc_state.prebuilt_objects = qcgc_shadow_stack_push(
+				qcgc_state.prebuilt_objects, object);
+	}
+
+	if (qcgc_state.phase == GC_PAUSE) {
+		return; // We are done
+	}
+
+	// Triggered barrier, we must not collect now
+	qcgc_state.phase = GC_MARK;
+
+	// Test reachability of object and push if neccessary
+	if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
+		// NOTE: No mark test here, as prebuilt objects are always reachable
+		// Push prebuilt object to general purpose gray stack
+		qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+				qcgc_state.gp_gray_stack, object);
+	} else if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
+		if (qcgc_hbtable_is_marked(object)) {
+			// Push huge block to general purpose gray stack
+			qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+					qcgc_state.gp_gray_stack, object);
+		}
+	} else {
+		if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK) {
+			// This was black before, push it to gray stack again
+			arena_t *arena = qcgc_arena_addr((cell_t *) object);
+			arena->gray_stack = qcgc_gray_stack_push(
+					arena->gray_stack, object);
 		}
 	}
 }
@@ -81,14 +119,31 @@
 	qcgc_event_logger_log(EVENT_ALLOCATE_START, sizeof(size_t),
 			(uint8_t *) &size);
 #endif
+	object_t *result;
+	if (size <= QCGC_LARGE_ALLOC_THRESHOLD) {
+		// Use bump / fit allocator
+		if (true) { // FIXME: Implement reasonable switch
+			result = qcgc_bump_allocate(size);
+		} else {
+			result = qcgc_fit_allocate(size);
 
-	object_t *result = (object_t *) qcgc_allocator_allocate(size);
-	result->flags |= QCGC_GRAY_FLAG;
+			// Fallback to bump allocator
+			if (result == NULL) {
+				result = qcgc_bump_allocate(size);
+			}
+		}
+	} else {
+		// Use huge block allocator
+		result = qcgc_large_allocate(size);
+	}
 
 #if LOG_ALLOCATION
 	qcgc_event_logger_log(EVENT_ALLOCATE_DONE, sizeof(object_t *),
 			(uint8_t *) &result);
 #endif
+#if CHECKED
+	assert(qcgc_state.phase != GC_COLLECT);
+#endif
 	return result;
 }
 
@@ -121,78 +176,69 @@
 	}
 }
 
-void qcgc_mark(void) {
-	qcgc_mark_all();
-}
-
-void qcgc_mark_all(void) {
+void qcgc_mark(bool incremental) {
 #if CHECKED
 	assert(qcgc_state.phase == GC_PAUSE || qcgc_state.phase == GC_MARK);
 #endif
+	// FIXME: Log some more information
 	qcgc_event_logger_log(EVENT_MARK_START, 0, NULL);
 
-	qcgc_state.phase = GC_MARK;
+	if (qcgc_state.phase == GC_PAUSE) {
+		qcgc_state.phase = GC_MARK;
 
-	// Push all roots
-	for (size_t i = 0; i < qcgc_state.shadow_stack->count; i++) {
-		qcgc_push_object(qcgc_state.shadow_stack->items[i]);
+		// If we do this for the first time, push all roots.
+		// All further changes to the roots (new additions) will be added
+		// by qcgc_shadowstack_push
+		for (size_t i = 0; i < qcgc_state.shadow_stack->count; i++) {
+			qcgc_push_object(qcgc_state.shadow_stack->items[i]);
+		}
+
+		// If we do this for the first time, push all prebuilt objects.
+		// All further changes to prebuilt objects will go to the gp_gray_stack
+		// because of the write barrier
+		size_t count = qcgc_state.prebuilt_objects->count;
+		for (size_t i = 0; i < count; i++) {
+			qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+					qcgc_state.gp_gray_stack,
+					qcgc_state.prebuilt_objects->items[i]);
+		}
 	}
 
-	// Trace all prebuilt objects
-	for (size_t i = 0; i < qcgc_state.prebuilt_objects->count; i++) {
-		qcgc_trace_cb(qcgc_state.prebuilt_objects->items[i], &qcgc_push_object);
-	}
+	while (qcgc_state.gray_stack_size > 0) {
+		// General purpose gray stack (prebuilt objects and huge blocks)
+		size_t to_process = (incremental ?
+			MIN(qcgc_state.gp_gray_stack->index,
+					MAX(qcgc_state.gp_gray_stack->index / 2, QCGC_INC_MARK_MIN)) :
+			(qcgc_state.gp_gray_stack->index));
 
-	while(qcgc_state.gray_stack_size > 0) {
+		while (to_process > 0) {
+			object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
+			qcgc_state.gp_gray_stack =
+				qcgc_gray_stack_pop(qcgc_state.gp_gray_stack);
+			qcgc_pop_object(top);
+			to_process--;
+		}
+
+		// Arena gray stacks
 		for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
 			arena_t *arena = qcgc_allocator_state.arenas->items[i];
-			while (arena->gray_stack->index > 0) {
+			to_process = (incremental ?
+					MIN(arena->gray_stack->index,
+						MAX(arena->gray_stack->index / 2, QCGC_INC_MARK_MIN)) :
+					(arena->gray_stack->index));
+
+			while (to_process > 0) {
 				object_t *top =
 					qcgc_gray_stack_top(arena->gray_stack);
 				arena->gray_stack =
 					qcgc_gray_stack_pop(arena->gray_stack);
 				qcgc_pop_object(top);
+				to_process--;
 			}
 		}
-	}
 
-	qcgc_state.phase = GC_COLLECT;
-
-	qcgc_event_logger_log(EVENT_MARK_DONE, 0, NULL);
-}
-
-void qcgc_mark_incremental(void) {
-#if CHECKED
-	assert(qcgc_state.phase == GC_PAUSE || qcgc_state.phase == GC_MARK);
-#endif
-	unsigned long gray_stack_size = qcgc_state.gray_stack_size;
-	qcgc_event_logger_log(EVENT_INCMARK_START, sizeof(gray_stack_size),
-			(uint8_t *) &gray_stack_size);
-
-	qcgc_state.phase = GC_MARK;
-
-	// Push all roots
-	for (size_t i = 0; i < qcgc_state.shadow_stack->count; i++) {
-		qcgc_push_object(qcgc_state.shadow_stack->items[i]);
-	}
-
-	// Trace all prebuilt objects
-	for (size_t i = 0; i < qcgc_state.prebuilt_objects->count; i++) {
-		qcgc_trace_cb(qcgc_state.prebuilt_objects->items[i], &qcgc_push_object);
-	}
-
-	for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
-		arena_t *arena = qcgc_allocator_state.arenas->items[i];
-		size_t initial_stack_size = arena->gray_stack->index;
-		size_t to_process = MIN(arena->gray_stack->index,
-				MAX(initial_stack_size / 2, QCGC_INC_MARK_MIN));
-		while (to_process > 0) {
-			object_t *top =
-				qcgc_gray_stack_top(arena->gray_stack);
-			arena->gray_stack =
-				qcgc_gray_stack_pop(arena->gray_stack);
-			qcgc_pop_object(top);
-			to_process--;
+		if (incremental) {
+			break; // Execute loop once for incremental collection
 		}
 	}
 
@@ -200,21 +246,29 @@
 		qcgc_state.phase = GC_COLLECT;
 	}
 
-	gray_stack_size = qcgc_state.gray_stack_size;
-	qcgc_event_logger_log(EVENT_INCMARK_START, sizeof(gray_stack_size),
-			(uint8_t *) &gray_stack_size);
+	// FIXME: Log some more information
+	qcgc_event_logger_log(EVENT_MARK_DONE, 0, NULL);
+#if CHECKED
+	assert(incremental || (qcgc_state.phase = GC_COLLECT));
+#endif
 }
 
 void qcgc_pop_object(object_t *object) {
 #if CHECKED
 	assert(object != NULL);
 	assert((object->flags & QCGC_GRAY_FLAG) == QCGC_GRAY_FLAG);
-	assert(qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK);
+	if (((object->flags & QCGC_PREBUILT_OBJECT) == 0) &&
+		((object_t *) qcgc_arena_addr((cell_t *) object) != object)) {
+		assert(qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK);
+	}
 #endif
 	object->flags &= ~QCGC_GRAY_FLAG;
 	qcgc_trace_cb(object, &qcgc_push_object);
 #if CHECKED
-	assert(qcgc_get_mark_color(object) == MARK_COLOR_BLACK);
+	if (((object->flags & QCGC_PREBUILT_OBJECT) == 0) &&
+		((object_t *) qcgc_arena_addr((cell_t *) object) != object)) {
+		assert(qcgc_get_mark_color(object) == MARK_COLOR_BLACK);
+	}
 #endif
 }
 
@@ -224,8 +278,17 @@
 	assert(qcgc_state.phase == GC_MARK);
 #endif
 	if (object != NULL) {
+		if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
+			if (qcgc_hbtable_mark(object)) {
+				// Did mark it / was white before
+				object->flags |= QCGC_GRAY_FLAG;
+				qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+						qcgc_state.gp_gray_stack, object);
+			}
+			return; // Skip tests
+		}
 		if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
-			return;
+			return; // Prebuilt objects are always black, no pushing here
 		}
 		if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_WHITE) {
 			object->flags |= QCGC_GRAY_FLAG;
@@ -258,6 +321,7 @@
 	qcgc_event_logger_log(EVENT_SWEEP_START, sizeof(arena_count),
 			(uint8_t *) &arena_count);
 
+	qcgc_hbtable_sweep();
 	for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
 		qcgc_arena_sweep(qcgc_allocator_state.arenas->items[i]);
 	}
@@ -267,6 +331,6 @@
 }
 
 void qcgc_collect(void) {
-	qcgc_mark();
+	qcgc_mark(false);
 	qcgc_sweep();
 }
diff --git a/rpython/translator/c/src/qcgc/shadow_stack.h b/rpython/translator/c/src/qcgc/shadow_stack.h
--- a/rpython/translator/c/src/qcgc/shadow_stack.h
+++ b/rpython/translator/c/src/qcgc/shadow_stack.h
@@ -12,8 +12,13 @@
 	object_t *items[];
 } shadow_stack_t;
 
+__attribute__ ((warn_unused_result))
 shadow_stack_t *qcgc_shadow_stack_create(size_t size);
 
+__attribute__ ((warn_unused_result))
 shadow_stack_t *qcgc_shadow_stack_push(shadow_stack_t *stack, object_t *item);
+
 object_t *qcgc_shadow_stack_top(shadow_stack_t *stack);
+
+__attribute__ ((warn_unused_result))
 shadow_stack_t *qcgc_shadow_stack_pop(shadow_stack_t *stack);

From pypy.commits at gmail.com  Sun Aug 28 05:55:34 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sun, 28 Aug 2016 02:55:34 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc compilation rule
Message-ID: <57c2b516.43681c0a.7915b.802f@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86637:b975e6511342
Date: 2016-08-27 16:41 +0200
http://bitbucket.org/pypy/pypy/changeset/b975e6511342/

Log:	Update qcgc compilation rule

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -900,7 +900,7 @@
             separate_module_sources = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
                 ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",
-                    "gray_stack.c", "shadow_stack.c"]],
+                    "gray_stack.c", "shadow_stack.c", "hugeblocktable.c"]],
             )
     return eci
 

From pypy.commits at gmail.com  Sun Aug 28 05:55:36 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sun, 28 Aug 2016 02:55:36 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Try implementing hash
Message-ID: <57c2b518.d4e41c0a.c038.8233@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86638:a1f8c2584b18
Date: 2016-08-28 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/a1f8c2584b18/

Log:	Try implementing hash

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -1,4 +1,5 @@
 from rpython.memory.gc.base import GCBase
+from rpython.memory.support import mangle_hash
 from rpython.rtyper.lltypesystem import rffi, lltype, llgroup, llmemory, llarena
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert
@@ -73,8 +74,21 @@
         #llop.gc_writebarrier(dest_addr)
         #return True
 
+    def id_or_identityhash(self, gcobj, is_hash):
+        i = self.header(llmemory.cast_ptr_to_adr(gcobj)).hash
+        prebuilt = llop.qcgc_is_prebuilt(lltype.Bool, gcobj)
+        #
+        if is_hash:
+            if prebuilt:
+                return i # Do not mangle for prebuilt objects
+            i = mangle_hash(i)
+        return i
+
+    def id(self, gcobje):
+        return self.id_or_identityhash(gcobj, False)
+
     def identityhash(self, gcobj):
-        raise NotImplementedError
+        return self.id_or_identityhash(gcobj, True)
 
     def register_finalizer(self, fq_index, gcobj):
         raise NotImplementedError
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -517,6 +517,7 @@
     # __________ qcgc operations __________
     'qcgc_allocate':    LLOp(canmallocgc=True),
     'qcgc_collect':     LLOp(canmallocgc=True),
+    'qcgc_is_prebuilt': LLOp(),
 
     # __________ weakrefs __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -959,3 +959,9 @@
 
     def OP_QCGC_COLLECT(self, op):
         return 'qcgc_collect();'
+
+    def OP_QCGC_IS_PREBUILT(self, op):
+        obj = self.expr(op.args[0])
+        result = self.expr(op.result)
+        return '%s = (((object_t *) %s)->flags & QCGC_PREBUILT_OBJECT) != 0;' % (
+                result, obj)
diff --git a/rpython/translator/c/src/qcgc/object.h b/rpython/translator/c/src/qcgc/object.h
--- a/rpython/translator/c/src/qcgc/object.h
+++ b/rpython/translator/c/src/qcgc/object.h
@@ -7,6 +7,7 @@
 #define QCGC_GRAY_FLAG (1<<0)
 #define QCGC_PREBUILT_OBJECT (1<<1)
 #define QCGC_PREBUILT_REGISTERED (1<<2)
+#define QCGC_FIRST_AVAILABLE_FLAG (1<<3)	// The first flag clients may use
 
 typedef struct object_s {
 	uint32_t flags;
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -43,9 +43,6 @@
  * Shadow stack
  */
 void qcgc_shadowstack_push(object_t *object) {
-#if CHECKED
-	assert((object->flags & QCGC_PREBUILT_OBJECT) == 0);
-#endif
 	if (qcgc_state.phase != GC_PAUSE) {
 		qcgc_state.phase = GC_MARK;
 		qcgc_push_object(object);

From pypy.commits at gmail.com  Sun Aug 28 07:26:01 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sun, 28 Aug 2016 04:26:01 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Implement correct hashing
Message-ID: <57c2ca49.465d1c0a.909b5.aa2e@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86639:551003ed3ffb
Date: 2016-08-28 13:25 +0200
http://bitbucket.org/pypy/pypy/changeset/551003ed3ffb/

Log:	Implement correct hashing

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -5,6 +5,9 @@
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.rarithmetic import ovfcheck
 
+QCGC_HAS_HASH = 0x100 # Upper half of flags for clients, lower half is reserved
+QCGC_PREBUILT_OBJECT = 0x2 # XXX: exploits knowledge about qcgc library
+
 class QCGC(GCBase):
     _alloc_flavor_ = "raw"
     moving_gc = False
@@ -16,20 +19,22 @@
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
 
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'hash', 0
+    withhash_flag_is_in_field = 'flags', QCGC_HAS_HASH
 
     TRANSLATION_PARAMS = {}
     HDR = lltype.Struct(
             'pypyhdr_t',
-            ('hdr', rffi.COpaque('object_t', hints={"is_qcgc_header": True})),
+            #('hdr', rffi.COpaque('object_t', hints={"is_qcgc_header": True})),
+            ('flags', lltype.Signed),   # XXX: exploits knowledge about object_t
             ('tid', lltype.Signed),
             ('hash', lltype.Signed))
     #HDR = rffi.COpaque('object_t')
 
-    def init_gc_object(self, obj, typeid):
-        hdr = llmemory.cast_adr_to_ptr(obj, lltype.Ptr(self.HDR))
+    def init_gc_object(self, addr, typeid, flags=0):
+        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+        hdr.flags = rffi.cast(lltype.Signed, flags)
         hdr.tid = rffi.cast(lltype.Signed, typeid)
-        hdr.hash = rffi.cast(lltype.Signed, obj)
+        hdr.hash = rffi.cast(lltype.Signed, addr)
 
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False,
@@ -59,6 +64,16 @@
         (obj + offset_to_length).signed[0] = length
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
+    def init_gc_object_immortal(self, addr, typeid, flags=0): # XXX: Prebuilt Objects?
+        assert flags == 0
+        ptr = self.gcheaderbuilder.object_from_header(addr.ptr)
+        prebuilt_hash = lltype.identityhash_nocache(ptr)
+        assert prebuilt_hash != 0
+        flags |= QCGC_PREBUILT_OBJECT
+        #
+        self.init_gc_object(addr, typeid.index, flags)
+        llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)).hash = prebuilt_hash
+
     def collect(self, gen=1):
         """Do a minor (gen=0) or major (gen>0) collection."""
         # XXX: Minor collection not supported
@@ -75,12 +90,13 @@
         #return True
 
     def id_or_identityhash(self, gcobj, is_hash):
-        i = self.header(llmemory.cast_ptr_to_adr(gcobj)).hash
-        prebuilt = llop.qcgc_is_prebuilt(lltype.Bool, gcobj)
+        hdr = self.header(llmemory.cast_ptr_to_adr(gcobj))
+        has_hash = (hdr.flags & QCGC_HAS_HASH)
+        i = hdr.hash
         #
         if is_hash:
-            if prebuilt:
-                return i # Do not mangle for prebuilt objects
+            if has_hash:
+                return i # Do not mangle for objects with built in hash
             i = mangle_hash(i)
         return i
 
@@ -95,27 +111,3 @@
 
     def get_type_id(self, obj):
         return self.header(obj).tid
-
-    def init_gc_object_immortal(self, addr, typeid, flags=0): # XXX: Prebuilt Objects?
-        assert flags == 0
-        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        hdr.tid = typeid.index
-        ptr = self.gcheaderbuilder.object_from_header(addr.ptr)
-        prebuilt_hash = lltype.identityhash_nocache(ptr)
-        assert prebuilt_hash != 0
-        #
-        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        hdr.hash = prebuilt_hash
-        #hdr._obj._name = typeid.index
-        #
-        # STMGC CODE:
-        #assert flags == 0
-        #assert isinstance(typeid16, llgroup.GroupMemberOffset)
-        #ptr = self.gcheaderbuilder.object_from_header(addr.ptr)
-        #prebuilt_hash = lltype.identityhash_nocache(ptr)
-        #assert prebuilt_hash != 0     # xxx probably good enough
-        ##
-        #hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        #hdr._obj._name = typeid16.index   # debug only
-        #hdr._obj.typeid16 = typeid16
-        #hdr._obj.prebuilt_hash = prebuilt_hash
diff --git a/rpython/translator/c/src/qcgc/object.h b/rpython/translator/c/src/qcgc/object.h
--- a/rpython/translator/c/src/qcgc/object.h
+++ b/rpython/translator/c/src/qcgc/object.h
@@ -7,7 +7,6 @@
 #define QCGC_GRAY_FLAG (1<<0)
 #define QCGC_PREBUILT_OBJECT (1<<1)
 #define QCGC_PREBUILT_REGISTERED (1<<2)
-#define QCGC_FIRST_AVAILABLE_FLAG (1<<3)	// The first flag clients may use
 
 typedef struct object_s {
 	uint32_t flags;

From pypy.commits at gmail.com  Sun Aug 28 08:30:48 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 28 Aug 2016 05:30:48 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Added tag release-pypy2.7-v5.4.0
 for changeset 68bb3510d821
Message-ID: <57c2d978.09afc20a.8df27.8e59@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86640:40da3e4a8ad5
Date: 2016-08-28 22:28 +1000
http://bitbucket.org/pypy/pypy/changeset/40da3e4a8ad5/

Log:	Added tag release-pypy2.7-v5.4.0 for changeset 68bb3510d821

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0

From pypy.commits at gmail.com  Sun Aug 28 08:32:13 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 28 Aug 2016 05:32:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: Added tag release-pypy2.7-v5.4.0 for
 changeset 68bb3510d821
Message-ID: <57c2d9cd.4219c20a.987dd.9249@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86641:0e7b60a80529
Date: 2016-08-28 22:28 +1000
http://bitbucket.org/pypy/pypy/changeset/0e7b60a80529/

Log:	Added tag release-pypy2.7-v5.4.0 for changeset 68bb3510d821

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0

From pypy.commits at gmail.com  Sun Aug 28 10:56:44 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sun, 28 Aug 2016 07:56:44 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Remove unnecessary define
Message-ID: <57c2fbac.02c41c0a.7d8c2.f88b@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86642:6fb3433530c7
Date: 2016-08-28 16:56 +0200
http://bitbucket.org/pypy/pypy/changeset/6fb3433530c7/

Log:	Remove unnecessary define

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -896,7 +896,6 @@
     eci = ExternalCompilationInfo(
             include_dirs = [library_dir],
             includes = ["qcgc.h"],
-            pre_include_bits = ["#define RPY_QCGC 1"],
             separate_module_sources = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
                 ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",

From pypy.commits at gmail.com  Sun Aug 28 12:26:39 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sun, 28 Aug 2016 09:26:39 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Remove assertions and just ignore
 finalizers
Message-ID: <57c310bf.94a51c0a.bea70.085a@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86643:a6e4ededebb0
Date: 2016-08-28 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/a6e4ededebb0/

Log:	Remove assertions and just ignore finalizers

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -40,10 +40,9 @@
                                needs_finalizer=False,
                                is_finalizer_light=False,
                                contains_weakptr=False):
-        # XXX: What is the llmemory.GCREF for? (Assumption: return value)
-        ll_assert(not needs_finalizer, 'finalizer not supported')
-        ll_assert(not is_finalizer_light, 'light finalizer not supported')
-        ll_assert(not contains_weakptr, 'weakref not supported')
+        #ll_assert(not needs_finalizer, 'finalizer not supported')
+        #ll_assert(not is_finalizer_light, 'light finalizer not supported')
+        #ll_assert(not contains_weakptr, 'weakref not supported')
         obj = llop.qcgc_allocate(llmemory.Address, size)
         self.init_gc_object(obj, typeid)
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -33,6 +33,20 @@
                   [SomeAddress(), SomePtr(VISIT_FPTR)],
                   s_None))
 
+    def gc_header_for(self, obj, needs_hash=False):
+        hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj)
+        withhash, flag = self.gcdata.gc.withhash_flag_is_in_field
+        x = getattr(hdr, withhash)
+        TYPE = lltype.typeOf(x)
+        x = lltype.cast_primitive(lltype.Signed, x)
+        if needs_hash:
+            x |= flag       # set the flag in the header
+        else:
+            x &= ~flag      # clear the flag in the header
+        x = lltype.cast_primitive(TYPE, x)
+        setattr(hdr, withhash, x)
+        return hdr
+
     def push_roots(self, hop, keep_current_args=False):
         livevars = self.get_livevars_for_roots(hop, keep_current_args)
         self.num_pushs += len(livevars)

From pypy.commits at gmail.com  Sun Aug 28 13:02:33 2016
From: pypy.commits at gmail.com (vext01)
Date: Sun, 28 Aug 2016 10:02:33 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Backed out changeset
 e50cf5d2ead3 -- Deal with OpenBSD tests elsewhere.
Message-ID: <57c31929.434bc20a.8dc9.9afe@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86644:c85b57ee51f7
Date: 2016-08-28 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/c85b57ee51f7/

Log:	Backed out changeset e50cf5d2ead3 -- Deal with OpenBSD tests
	elsewhere.

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1,5 +1,4 @@
 import py, sys, random, os, struct, operator
-import pytest
 from rpython.jit.metainterp.history import (AbstractFailDescr,
                                          AbstractDescr,
                                          BasicFailDescr,
@@ -2600,8 +2599,6 @@
         deadframe2 = self.cpu.force(frame)
         assert self.cpu.get_int_value(deadframe2, 0) == 30
 
-    @pytest.mark.xfail(sys.platform.startswith("openbsd"),
-                       reason="something wrong with CDLL()")
     def test_call_to_c_function(self):
         from rpython.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL
         from rpython.rtyper.lltypesystem.ll2ctypes import libc_name
@@ -2628,8 +2625,6 @@
         assert fail.identifier == 0
         assert self.cpu.get_int_value(deadframe, 0) == ord('g')
 
-    @pytest.mark.xfail(sys.platform.startswith("openbsd"),
-                       reason="something wrong with CDLL()")
     def test_call_to_c_function_with_callback(self):
         from rpython.rlib.libffi import CDLL, types, ArgChain, clibffi
         from rpython.rtyper.lltypesystem.ll2ctypes import libc_name
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -17,7 +17,6 @@
 _MAC_OS = platform.name == "darwin"
 _FREEBSD = sys.platform.startswith("freebsd")
 _NETBSD = sys.platform.startswith("netbsd")
-_OPENBSD = sys.platform.startswith("openbsd")
 
 if _WIN32:
     from rpython.rlib import rwin32
@@ -30,7 +29,7 @@
 else:
     pre_include_bits = []
 
-if _OPENBSD or _FREEBSD or _NETBSD or _WIN32:
+if _FREEBSD or _NETBSD or _WIN32:
     libraries = []
 else:
     libraries = ['dl']

From pypy.commits at gmail.com  Sun Aug 28 13:02:35 2016
From: pypy.commits at gmail.com (vext01)
Date: Sun, 28 Aug 2016 10:02:35 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Backed out changeset
 2bcb6534dd1c -- Deal with OpenBSD tests elsewhere.
Message-ID: <57c3192b.8a13c20a.23ed6.d856@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86645:4c9888c08526
Date: 2016-08-28 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/4c9888c08526/

Log:	Backed out changeset 2bcb6534dd1c -- Deal with OpenBSD tests
	elsewhere.

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -4476,11 +4476,6 @@
                 expected = heaptracker.int_signext(test_case, numbytes)
                 assert got == expected
 
-    @pytest.mark.xfail(sys.platform.startswith("openbsd"),
-                       reason="emits wrong code, or objdump reports wrong."
-                       "looks like openbsd objdump doesn't understand NOPs "
-                       "with operands. e.g. 0f1f00 is 'nop DWORD PTR [rax]', "
-                       "but we just get '(bad)'. Old binutils issue?")
     def test_compile_asmlen(self):
         if not isinstance(self.cpu, AbstractLLCPU):
             py.test.skip("pointless test on non-asm")
diff --git a/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_64_auto_encoding.py
@@ -3,8 +3,6 @@
 from rpython.jit.backend.x86.test import test_rx86_32_auto_encoding
 
 
-# XXX OpenBSD
-# Many unrecognised asm instrs. Probably due to openbsd's old binutils
 class TestRx86_64(test_rx86_32_auto_encoding.TestRx86_32):
     WORD = 8
     TESTDIR = 'rx86_64'
diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py
--- a/rpython/jit/backend/x86/test/test_zrpy_gc.py
+++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py
@@ -1,16 +1,6 @@
 from rpython.jit.backend.llsupport.test.zrpy_gc_test import CompileFrameworkTests
-import pytest
 
 
-# XXX OpenBSD
-# Generated binary crashes:
-# mem.c: 13 mallocs left (use PYPY_ALLOC=1 to see the list)
-# RPython traceback:
-#   File "rpython_jit_backend_llsupport_test.c", line 232, in entrypoint
-#   File "rpython_jit_backend_llsupport_test.c", line 520, in allfuncs
-#   File "rpython_jit_backend_llsupport_test.c", line 599, in main_allfuncs
-#   File "rpython_rtyper_lltypesystem.c", line 3658, in ll_dict_getitem_with_hash__dicttablePtr_rpy_stri
-# Fatal RPython error: KeyError
 class TestShadowStack(CompileFrameworkTests):
     gcrootfinder = "shadowstack"
     gc = "incminimark"
diff --git a/rpython/jit/backend/zarch/test/test_auto_encoding.py b/rpython/jit/backend/zarch/test/test_auto_encoding.py
--- a/rpython/jit/backend/zarch/test/test_auto_encoding.py
+++ b/rpython/jit/backend/zarch/test/test_auto_encoding.py
@@ -285,9 +285,6 @@
         if not data.startswith('GNU assembler'):
             py.test.skip("full tests require the GNU 'as' assembler")
 
-
-    # XXX OpenBSD
-    # Many unrecognised asm instrs. Probably due to openbsd's old binutils
     @py.test.mark.parametrize("name", codebuilder.all_instructions)
     def test_all(self, name):
         self.complete_test(name)

From pypy.commits at gmail.com  Sun Aug 28 14:01:26 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 28 Aug 2016 11:01:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: missing include (osx)
Message-ID: <57c326f6.17a71c0a.aea9e.28b1@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r86646:424353c8a77e
Date: 2016-08-28 10:54 -0700
http://bitbucket.org/pypy/pypy/changeset/424353c8a77e/

Log:	missing include (osx)

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -2142,6 +2142,7 @@
 eci_inheritable = eci.merge(ExternalCompilationInfo(
     separate_module_sources=[r"""
 #include <errno.h>
+#include <sys/ioctl.h>
 
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)

From pypy.commits at gmail.com  Sun Aug 28 14:01:28 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 28 Aug 2016 11:01:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix translation (win32)
Message-ID: <57c326f8.e129c20a.4ccf.eeaa@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r86647:bfdd07da8ed7
Date: 2016-08-28 11:00 -0700
http://bitbucket.org/pypy/pypy/changeset/bfdd07da8ed7/

Log:	fix translation (win32)

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1172,14 +1172,11 @@
         if ok:
             fdread = c_open_osfhandle(hread, 0)
             fdwrite = c_open_osfhandle(hwrite, 1)
-            if fdread == -1 or fdwrite == -1:
-                rwin32.CloseHandle(hread)
-                rwin32.CloseHandle(hwrite)
-                ok = 0
-        if not ok:
-            raise WindowsError(rwin32.GetLastError_saved(),
-                               "CreatePipe failed")
-        return (fdread, fdwrite)
+            if not (fdread == -1 or fdwrite == -1):
+                return (fdread, fdwrite)
+            rwin32.CloseHandle(hread)
+            rwin32.CloseHandle(hwrite)
+        raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:

From pypy.commits at gmail.com  Sun Aug 28 14:07:13 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 28 Aug 2016 11:07:13 -0700 (PDT)
Subject: [pypy-commit] buildbot default: build py3k nightly on jitwin32
Message-ID: <57c32851.041f1c0a.8d1b3.213b@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r1016:cd282179b97c
Date: 2016-08-28 11:06 -0700
http://bitbucket.org/pypy/buildbot/changeset/cd282179b97c/

Log:	build py3k nightly on jitwin32

diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -325,6 +325,7 @@
             LINUX64,                   # on bencher4, uses all cores
             JITLINUX64,                # on bencher4, uses 1 core
             JITMACOSX64,               # on xerxes
+            JITWIN32,                  # on allegro_win32, SalsaSalsa
             ], branch="py3k", hour=4, minute=0),
 
         # S390X vm (ibm-research)

From pypy.commits at gmail.com  Sun Aug 28 14:14:06 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 28 Aug 2016 11:14:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: work on the handle
Message-ID: <57c329ee.c3f0c20a.cb08.f492@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r86648:304ab96afc72
Date: 2016-08-28 11:13 -0700
http://bitbucket.org/pypy/pypy/changeset/304ab96afc72/

Log:	work on the handle

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1174,8 +1174,8 @@
             fdwrite = c_open_osfhandle(hwrite, 1)
             if not (fdread == -1 or fdwrite == -1):
                 return (fdread, fdwrite)
-            rwin32.CloseHandle(hread)
-            rwin32.CloseHandle(hwrite)
+            rwin32.CloseHandle(pread)
+            rwin32.CloseHandle(pwrite)
         raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')

From pypy.commits at gmail.com  Sun Aug 28 14:47:00 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 28 Aug 2016 11:47:00 -0700 (PDT)
Subject: [pypy-commit] pypy rpython-resync: Update for changed signature of
 pypysig_set_wakeup_fd()
Message-ID: <57c331a4.497bc20a.5c897.f72a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rpython-resync
Changeset: r86649:52f419ecbd77
Date: 2016-08-28 19:44 +0100
http://bitbucket.org/pypy/pypy/changeset/52f419ecbd77/

Log:	Update for changed signature of pypysig_set_wakeup_fd()

diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -64,7 +64,7 @@
         AsyncAction.__init__(self, space)
         self.pending_signal = -1
         self.fire_in_another_thread = False
-        #
+
         @rgc.no_collect
         def _after_thread_switch():
             if self.fire_in_another_thread:
@@ -251,7 +251,7 @@
         except OSError as e:
             if e.errno == errno.EBADF:
                 raise oefmt(space.w_ValueError, "invalid fd")
-    old_fd = pypysig_set_wakeup_fd(fd)
+    old_fd = pypysig_set_wakeup_fd(fd, True)
     return space.wrap(intmask(old_fd))
 
 

From pypy.commits at gmail.com  Sun Aug 28 15:46:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 Aug 2016 12:46:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2382: fixed the import logic.
 The main difference is that if we
Message-ID: <57c33fa3.54bc1c0a.7f9a9.509c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86650:9332dfa6e22e
Date: 2016-08-28 21:08 +0200
http://bitbucket.org/pypy/pypy/changeset/9332dfa6e22e/

Log:	Issue #2382: fixed the import logic. The main difference is that if
	we call an import hook via sys.meta_path, it succeeds (a bit
	brokenly, like CPython) even if the load_module() method doesn't add
	the module in sys.modules.

diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -123,5 +123,4 @@
         pathname = code.co_filename
     w_mod = importing.add_module(space, w_name)
     space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname))
-    importing.exec_code_module(space, w_mod, code)
-    return w_mod
+    return importing.exec_code_module(space, w_mod, code, w_name)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -597,6 +597,11 @@
 
 @jit.dont_look_inside
 def load_module(space, w_modulename, find_info, reuse=False):
+    """Like load_module() in CPython's import.c, this will normally
+    make a module object, store it in sys.modules, execute code in it,
+    and then fetch it again from sys.modules.  But this logic is not
+    used if we're calling a PEP302 loader.
+    """
     if find_info is None:
         return
 
@@ -625,17 +630,15 @@
 
         try:
             if find_info.modtype == PY_SOURCE:
-                load_source_module(
+                return load_source_module(
                     space, w_modulename, w_mod,
                     find_info.filename, find_info.stream.readall(),
                     find_info.stream.try_to_find_file_descriptor())
-                return w_mod
             elif find_info.modtype == PY_COMPILED:
                 magic = _r_long(find_info.stream)
                 timestamp = _r_long(find_info.stream)
-                load_compiled_module(space, w_modulename, w_mod, find_info.filename,
+                return load_compiled_module(space, w_modulename, w_mod, find_info.filename,
                                      magic, timestamp, find_info.stream.readall())
-                return w_mod
             elif find_info.modtype == PKG_DIRECTORY:
                 w_path = space.newlist([space.wrap(find_info.filename)])
                 space.setattr(w_mod, space.wrap('__path__'), w_path)
@@ -644,14 +647,13 @@
                 if find_info is None:
                     return w_mod
                 try:
-                    load_module(space, w_modulename, find_info, reuse=True)
+                    w_mod = load_module(space, w_modulename, find_info,
+                                        reuse=True)
                 finally:
                     try:
                         find_info.stream.close()
                     except StreamErrors:
                         pass
-                # fetch the module again, in case of "substitution"
-                w_mod = check_sys_modules(space, w_modulename)
                 return w_mod
             elif find_info.modtype == C_EXTENSION and has_so_extension(space):
                 load_c_extension(space, find_info.filename, space.str_w(w_modulename))
@@ -677,13 +679,6 @@
         try:
             if find_info:
                 w_mod = load_module(space, w_modulename, find_info)
-                try:
-                    w_mod = space.getitem(space.sys.get("modules"),
-                                          w_modulename)
-                except OperationError as oe:
-                    if not oe.match(space, space.w_KeyError):
-                        raise
-                    raise OperationError(space.w_ImportError, w_modulename)
                 if w_parent is not None:
                     space.setattr(w_parent, space.wrap(partname), w_mod)
                 return w_mod
@@ -875,20 +870,32 @@
     pycode = ec.compiler.compile(source, pathname, 'exec', 0)
     return pycode
 
-def exec_code_module(space, w_mod, code_w):
+def exec_code_module(space, w_mod, code_w, w_modulename, check_afterwards=True):
+    """
+    Execute a code object in the module's dict.  Returns
+    'sys.modules[modulename]', which must exist.
+    """
     w_dict = space.getattr(w_mod, space.wrap('__dict__'))
     space.call_method(w_dict, 'setdefault',
                       space.wrap('__builtins__'),
                       space.wrap(space.builtin))
     code_w.exec_code(space, w_dict, w_dict)
 
+    if check_afterwards:
+        w_mod = check_sys_modules(space, w_modulename)
+        if w_mod is None:
+            raise oefmt(space.w_ImportError,
+                        "Loaded module %R not found in sys.modules",
+                        w_modulename)
+    return w_mod
+
 
 @jit.dont_look_inside
 def load_source_module(space, w_modulename, w_mod, pathname, source, fd,
-                       write_pyc=True):
+                       write_pyc=True, check_afterwards=True):
     """
-    Load a source module from a given file and return its module
-    object.
+    Load a source module from a given file.  Returns the result
+    of sys.modules[modulename], which must exist.
     """
     w = space.wrap
 
@@ -927,9 +934,8 @@
         code_w.remove_docstrings(space)
 
     update_code_filenames(space, code_w, pathname)
-    exec_code_module(space, w_mod, code_w)
-
-    return w_mod
+    return exec_code_module(space, w_mod, code_w, w_modulename,
+                            check_afterwards=check_afterwards)
 
 def update_code_filenames(space, code_w, pathname, oldname=None):
     assert isinstance(code_w, PyCode)
@@ -1012,10 +1018,10 @@
 
 @jit.dont_look_inside
 def load_compiled_module(space, w_modulename, w_mod, cpathname, magic,
-                         timestamp, source):
+                         timestamp, source, check_afterwards=True):
     """
-    Load a module from a compiled file, execute it, and return its
-    module object.
+    Load a module from a compiled file and execute it.  Returns
+    'sys.modules[modulename]', which must exist.
     """
     log_pyverbose(space, 1, "import %s # compiled from %s\n" %
                   (space.str_w(w_modulename), cpathname))
@@ -1032,9 +1038,8 @@
     if optimize >= 2:
         code_w.remove_docstrings(space)
 
-    exec_code_module(space, w_mod, code_w)
-
-    return w_mod
+    return exec_code_module(space, w_mod, code_w, w_modulename,
+                            check_afterwards=check_afterwards)
 
 def open_exclusive(space, cpathname, mode):
     try:
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -98,33 +98,35 @@
     w_mod = space.wrap(Module(space, w_modulename))
     importing._prepare_module(space, w_mod, filename, None)
 
-    importing.load_source_module(
+    w_mod = importing.load_source_module(
         space, w_modulename, w_mod,
         filename, stream.readall(), stream.try_to_find_file_descriptor())
     if space.is_none(w_file):
         stream.close()
     return w_mod
 
- at unwrap_spec(filename='str0')
-def _run_compiled_module(space, w_modulename, filename, w_file, w_module):
+ at unwrap_spec(filename='str0', check_afterwards=int)
+def _run_compiled_module(space, w_modulename, filename, w_file, w_module,
+                         check_afterwards=False):
     # the function 'imp._run_compiled_module' is a pypy-only extension
     stream = get_file(space, w_file, filename, 'rb')
 
     magic = importing._r_long(stream)
     timestamp = importing._r_long(stream)
 
-    importing.load_compiled_module(
+    w_mod = importing.load_compiled_module(
         space, w_modulename, w_module, filename, magic, timestamp,
-        stream.readall())
+        stream.readall(), check_afterwards=check_afterwards)
     if space.is_none(w_file):
         stream.close()
+    return w_mod
 
 @unwrap_spec(filename='str0')
 def load_compiled(space, w_modulename, filename, w_file=None):
     w_mod = space.wrap(Module(space, w_modulename))
     importing._prepare_module(space, w_mod, filename, None)
-    _run_compiled_module(space, w_modulename, filename, w_file, w_mod)
-    return w_mod
+    return _run_compiled_module(space, w_modulename, filename, w_file, w_mod,
+                                check_afterwards=True)
 
 @unwrap_spec(filename=str)
 def load_dynamic(space, w_modulename, filename, w_file=None):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -118,7 +118,7 @@
         filename = str(p.join("x.py"))
         stream = streamio.open_file_as_stream(filename, "r")
         try:
-            importing.load_source_module(
+            _load_source_module(
                 space, w_modname, w(importing.Module(space, w_modname)),
                 filename, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -139,6 +139,15 @@
 
     return str(root)
 
+def _load_source_module(space, w_modname, w_mod, *args, **kwds):
+    kwds.setdefault('check_afterwards', False)
+    return importing.load_source_module(space, w_modname, w_mod, *args, **kwds)
+
+def _load_compiled_module(space, w_modname, w_mod, *args, **kwds):
+    kwds.setdefault('check_afterwards', False)
+    return importing.load_compiled_module(space, w_modname, w_mod,
+                                          *args, **kwds)
+
 
 def _setup(space):
     dn = setup_directory_structure(space)
@@ -887,8 +896,7 @@
             w_mod = space.wrap(Module(space, w_modulename))
             magic = importing._r_long(stream)
             timestamp = importing._r_long(stream)
-            w_ret = importing.load_compiled_module(space,
-                                                   w_modulename,
+            w_ret = _load_compiled_module(space,   w_modulename,
                                                    w_mod,
                                                    cpathname,
                                                    magic,
@@ -946,7 +954,7 @@
         pathname = _testfilesource()
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -968,7 +976,7 @@
         pathname = _testfilesource()
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor(),
@@ -987,7 +995,7 @@
         try:
             space.setattr(space.sys, space.wrap('dont_write_bytecode'),
                           space.w_True)
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -1006,7 +1014,7 @@
         pathname = _testfilesource(source="<Syntax Error>")
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -1026,7 +1034,7 @@
         pathname = _testfilesource(source="a = unknown_name")
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -1114,7 +1122,7 @@
                     magic = importing._r_long(stream)
                     timestamp = importing._r_long(stream)
                     space2.raises_w(space2.w_ImportError,
-                                    importing.load_compiled_module,
+                                    _load_compiled_module,
                                     space2,
                                     w_modulename,
                                     w_mod,
@@ -1326,10 +1334,7 @@
         # use an import hook that doesn't update sys.modules, then the
         # import succeeds; but at the same time, you can have the same
         # result without an import hook (see test_del_from_sys_modules)
-        # and then the import fails.  This looks like even more mess
-        # to replicate, so we ignore it until someone really hits this
-        # case...
-        skip("looks like an inconsistency in CPython")
+        # and then the import fails.  Mess mess mess.
 
         class ImportHook(object):
             def find_module(self, fullname, path=None):
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -152,8 +152,7 @@
         importing._prepare_module(space, w_mod, real_name, pkgpath)
         co_filename = self.make_co_filename(filename)
         code_w = importing.parse_source_module(space, co_filename, buf)
-        importing.exec_code_module(space, w_mod, code_w)
-        return w_mod
+        return importing.exec_code_module(space, w_mod, code_w, w(modname))
 
     def _parse_mtime(self, space, filename):
         w = space.wrap
@@ -205,10 +204,10 @@
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
         space.setattr(w_mod, w('__loader__'), space.wrap(self))
         importing._prepare_module(space, w_mod, real_name, pkgpath)
-        result = importing.load_compiled_module(space, w(modname), w_mod,
+        w_result = importing.load_compiled_module(space, w(modname), w_mod,
                                                 filename, magic, timestamp,
                                                 buf)
-        return result
+        return w_result
 
     def have_modulefile(self, space, filename):
         if ZIPSEP != os.path.sep:

From pypy.commits at gmail.com  Sun Aug 28 15:46:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 Aug 2016 12:46:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <57c33fa5.436ec20a.d0586.185e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86651:0f0d92ccb2c1
Date: 2016-08-28 21:46 +0200
http://bitbucket.org/pypy/pypy/changeset/0f0d92ccb2c1/

Log:	merge heads

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,6 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 4176c6f63109
+.. startrev: 522736f816dc
 
 
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=3
-rev=1
+min=4
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Sun Aug 28 15:47:58 2016
From: pypy.commits at gmail.com (vext01)
Date: Sun, 28 Aug 2016 12:47:58 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Fix another W^X
 violation in test logic.
Message-ID: <57c33fee.262ec20a.3aef8.14b6@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86654:8f8a16f423e9
Date: 2016-08-28 20:46 +0100
http://bitbucket.org/pypy/pypy/changeset/8f8a16f423e9/

Log:	Fix another W^X violation in test logic.

diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -301,8 +301,11 @@
         self.get_relative_pos = self._data.__len__
         def plain_copy_to_raw_memory(addr):
             dst = rffi.cast(rffi.CCHARP, addr)
+            size = len(self._data)
+            rmmap.set_pages_writable(dst, size)
             for i, c in enumerate(self._data):
                 dst[i] = c
+            rmmap.set_pages_executable(dst, size)
         self._copy_to_raw_memory = plain_copy_to_raw_memory
 
     def insert_gcroot_marker(self, mark):

From pypy.commits at gmail.com  Sun Aug 28 15:48:00 2016
From: pypy.commits at gmail.com (vext01)
Date: Sun, 28 Aug 2016 12:48:00 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: This test should be
 using mmap.
Message-ID: <57c33ff0.4825c20a.df1f5.4879@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86655:6e31d839fd03
Date: 2016-08-28 20:47 +0100
http://bitbucket.org/pypy/pypy/changeset/6e31d839fd03/

Log:	This test should be using mmap.

diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -4,6 +4,7 @@
 from rpython.jit.backend.llsupport.codemap import CodemapStorage
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
+from rpython.rlib import rmmap
 
 
 def test_get_index():
@@ -188,7 +189,8 @@
 def test_blockbuildermixin(translated=True):
     mc = BlockBuilderMixin(translated)
     writtencode = []
-    for i in range(mc.SUBBLOCK_SIZE * 2 + 3):
+    map_size = mc.SUBBLOCK_SIZE * 2 + 3
+    for i in range(map_size):
         assert mc.get_relative_pos() == i
         mc.writechar(chr(i % 255))
         writtencode.append(chr(i % 255))
@@ -203,11 +205,11 @@
         mc.overwrite(i, chr((i + 63) % 255))
         writtencode[i] = chr((i + 63) % 255)
     #
-    p = lltype.malloc(rffi.CCHARP.TO, mc.SUBBLOCK_SIZE * 2 + 3, flavor='raw')
+    p = rmmap.alloc(map_size)
     addr = rffi.cast(lltype.Signed, p)
     mc.copy_to_raw_memory(addr)
     #
-    for i in range(mc.SUBBLOCK_SIZE * 2 + 3):
+    for i in range(map_size):
         assert p[i] == writtencode[i]
     #
     debug._log = debug.DebugLog()
@@ -222,7 +224,7 @@
                     [('debug_print', 'SYS_EXECUTABLE', '??'),
                      ('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
     
-    lltype.free(p, flavor='raw')
+    rmmap.free(p, map_size)
 
 def test_blockbuildermixin2():
     test_blockbuildermixin(translated=False)

From pypy.commits at gmail.com  Sun Aug 28 15:47:55 2016
From: pypy.commits at gmail.com (vext01)
Date: Sun, 28 Aug 2016 12:47:55 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Fix assertions in
 rmmap.set_pages_*
Message-ID: <57c33feb.11331c0a.14c66.4709@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86652:2a5b8eb3457c
Date: 2016-08-28 20:33 +0100
http://bitbucket.org/pypy/pypy/changeset/2a5b8eb3457c/

Log:	Fix assertions in rmmap.set_pages_*

diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -727,7 +727,7 @@
         return c_mprotect_safe(addr, size, prot)
 
     def set_pages_executable(addr, size):
-        assert isinstance(addr, lltype._ptr)
+        assert lltype.typeOf(addr) == rffi.CCHARP
         assert isinstance(size, int)
         rv = mprotect(addr, size, PROT_EXEC | PROT_READ)
         if int(rv) < 0:
@@ -735,7 +735,7 @@
             debug.fatalerror_notb("set_pages_executable failed")
 
     def set_pages_writable(addr, size):
-        assert isinstance(addr, lltype._ptr)
+        assert lltype.typeOf(addr) == rffi.CCHARP
         assert isinstance(size, int)
         rv = mprotect(addr, size, PROT_WRITE | PROT_READ)
         if int(rv) < 0:

From pypy.commits at gmail.com  Sun Aug 28 15:47:56 2016
From: pypy.commits at gmail.com (vext01)
Date: Sun, 28 Aug 2016 12:47:56 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Align mprotect on Linux.
Message-ID: <57c33fec.88711c0a.70789.4fd8@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86653:0f9c35877fce
Date: 2016-08-28 20:33 +0100
http://bitbucket.org/pypy/pypy/changeset/0f9c35877fce/

Log:	Align mprotect on Linux.

diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -23,6 +23,11 @@
 _MS_WINDOWS = os.name == "nt"
 _64BIT = "64bit" in platform.architecture()[0]
 _CYGWIN = "cygwin" == sys.platform
+_LINUX = sys.platform.startswith("linux")
+
+if _LINUX:
+    # XXX use getconf to query the actual page size
+    PAGE_SIZE = 4096
 
 class RMMapError(Exception):
     def __init__(self, message):
@@ -724,7 +729,19 @@
         addr = rffi.cast(PTR, addr)
         size = rffi.cast(lltype.Unsigned, size)
         prot = rffi.cast(rffi.INT, prot)
-        return c_mprotect_safe(addr, size, prot)
+
+        # The start address (addr) has to be page aligned on Linux
+        if _LINUX:
+            # XXX optimise?
+            addr_int = rffi.cast(rffi.UINTPTR_T, addr)
+            real_addr_int = (addr_int / PAGE_SIZE) * PAGE_SIZE
+            real_addr = rffi.cast(rffi.CCHARP, real_addr_int)
+            real_size = size + (addr_int - real_addr_int)
+        else:
+            real_addr = addr
+            real_size = size
+
+        return c_mprotect_safe(real_addr, real_size, prot)
 
     def set_pages_executable(addr, size):
         assert lltype.typeOf(addr) == rffi.CCHARP

From pypy.commits at gmail.com  Sun Aug 28 16:14:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 Aug 2016 13:14:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c3462c.e97ac20a.936fc.178f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86656:b1f759cbec09
Date: 2016-08-28 21:58 +0200
http://bitbucket.org/pypy/pypy/changeset/b1f759cbec09/

Log:	hg merge default

	(in particular, 9332dfa6e22e does not apply any more)

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,6 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 4176c6f63109
+.. startrev: 522736f816dc
 
 
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1188,14 +1188,8 @@
             sys.path_hooks.pop()
 
     def test_meta_path_import_error_1(self):
-        # as far as I can tell, the problem is that in CPython, if you
-        # use an import hook that doesn't update sys.modules, then the
-        # import succeeds; but at the same time, you can have the same
-        # result without an import hook (see test_del_from_sys_modules)
-        # and then the import fails.  This looks like even more mess
-        # to replicate, so we ignore it until someone really hits this
-        # case...
-        skip("looks like an inconsistency in CPython")
+        # check that we get a KeyError somewhere inside
+        # <frozen importlib._bootstrap>, like CPython 3.5
 
         class ImportHook(object):
             def find_module(self, fullname, path=None):
@@ -1205,12 +1199,12 @@
             def load_module(self, fullname):
                 assert fullname == 'meta_path_pseudo_module'
                 # we "forget" to update sys.modules
-                return new.module('meta_path_pseudo_module')
+                return types.ModuleType('meta_path_pseudo_module')
 
-        import sys, new
+        import sys, types
         sys.meta_path.append(ImportHook())
         try:
-            import meta_path_pseudo_module
+            raises(KeyError, "import meta_path_pseudo_module")
         finally:
             sys.meta_path.pop()
 
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=3
-rev=1
+min=4
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Sun Aug 28 16:14:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 Aug 2016 13:14:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: translation fixes
Message-ID: <57c34631.05d71c0a.351b2.fa93@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86658:6fbcade1c6d6
Date: 2016-08-28 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/6fbcade1c6d6/

Log:	translation fixes

diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -11,7 +11,8 @@
 # objects, are supported.  Version 3 of this protocol properly
 # supports circular links and sharing.  The previous version is called
 # "2", like in Python 2.7, although it is not always compatible
-# between CPython 2.7 and CPython 3.4.
+# between CPython 2.7 and CPython 3.4.  Version 4 adds small
+# optimizations in compactness.
 #
 # XXX: before py3k, there was logic to do efficiently dump()/load() on
 # a file object.  The corresponding logic is gone from CPython 3.x, so
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -413,7 +413,7 @@
     varnames    = _unmarshal_strlist(u)
     freevars    = _unmarshal_strlist(u)
     cellvars    = _unmarshal_strlist(u)
-    filename    = space.unicode_w(u.get_w_obj()).encode('utf-8')
+    filename    = space.unicode0_w(u.get_w_obj()).encode('utf-8')
     name        = space.unicode_w(u.get_w_obj()).encode('utf-8')
     firstlineno = u.get_int()
     lnotab      = space.bytes_w(u.get_w_obj())
@@ -460,7 +460,7 @@
     else:
         lng = u.get_lng()
     s = u.get(lng)
-    w_u = u.space.newunicode(s.decode('latin1'))
+    w_u = u.space.newunicode(s.decode('latin-1'))
     if interned:
         w_u = u.space.new_interned_w_str(w_u)
     return w_u

From pypy.commits at gmail.com  Sun Aug 28 16:14:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 Aug 2016 13:14:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: hg merge py3.5
Message-ID: <57c34632.88cb1c0a.7e887.5796@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86659:758d9fa7c013
Date: 2016-08-28 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/758d9fa7c013/

Log:	hg merge py3.5

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,6 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 4176c6f63109
+.. startrev: 522736f816dc
 
 
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1188,14 +1188,8 @@
             sys.path_hooks.pop()
 
     def test_meta_path_import_error_1(self):
-        # as far as I can tell, the problem is that in CPython, if you
-        # use an import hook that doesn't update sys.modules, then the
-        # import succeeds; but at the same time, you can have the same
-        # result without an import hook (see test_del_from_sys_modules)
-        # and then the import fails.  This looks like even more mess
-        # to replicate, so we ignore it until someone really hits this
-        # case...
-        skip("looks like an inconsistency in CPython")
+        # check that we get a KeyError somewhere inside
+        # <frozen importlib._bootstrap>, like CPython 3.5
 
         class ImportHook(object):
             def find_module(self, fullname, path=None):
@@ -1205,12 +1199,12 @@
             def load_module(self, fullname):
                 assert fullname == 'meta_path_pseudo_module'
                 # we "forget" to update sys.modules
-                return new.module('meta_path_pseudo_module')
+                return types.ModuleType('meta_path_pseudo_module')
 
-        import sys, new
+        import sys, types
         sys.meta_path.append(ImportHook())
         try:
-            import meta_path_pseudo_module
+            raises(KeyError, "import meta_path_pseudo_module")
         finally:
             sys.meta_path.pop()
 
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=3
-rev=1
+min=4
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1172,14 +1172,11 @@
         if ok:
             fdread = c_open_osfhandle(hread, 0)
             fdwrite = c_open_osfhandle(hwrite, 1)
-            if fdread == -1 or fdwrite == -1:
-                rwin32.CloseHandle(hread)
-                rwin32.CloseHandle(hwrite)
-                ok = 0
-        if not ok:
-            raise WindowsError(rwin32.GetLastError_saved(),
-                               "CreatePipe failed")
-        return (fdread, fdwrite)
+            if not (fdread == -1 or fdwrite == -1):
+                return (fdread, fdwrite)
+            rwin32.CloseHandle(pread)
+            rwin32.CloseHandle(pwrite)
+        raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
@@ -2142,6 +2139,7 @@
 eci_inheritable = eci.merge(ExternalCompilationInfo(
     separate_module_sources=[r"""
 #include <errno.h>
+#include <sys/ioctl.h>
 
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)

From pypy.commits at gmail.com  Sun Aug 28 16:14:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 Aug 2016 13:14:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c3462e.c997c20a.2d8c4.2347@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86657:69b5503f284b
Date: 2016-08-28 22:12 +0200
http://bitbucket.org/pypy/pypy/changeset/69b5503f284b/

Log:	hg merge py3k

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,6 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 4176c6f63109
+.. startrev: 522736f816dc
 
 
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1188,14 +1188,8 @@
             sys.path_hooks.pop()
 
     def test_meta_path_import_error_1(self):
-        # as far as I can tell, the problem is that in CPython, if you
-        # use an import hook that doesn't update sys.modules, then the
-        # import succeeds; but at the same time, you can have the same
-        # result without an import hook (see test_del_from_sys_modules)
-        # and then the import fails.  This looks like even more mess
-        # to replicate, so we ignore it until someone really hits this
-        # case...
-        skip("looks like an inconsistency in CPython")
+        # check that we get a KeyError somewhere inside
+        # <frozen importlib._bootstrap>, like CPython 3.5
 
         class ImportHook(object):
             def find_module(self, fullname, path=None):
@@ -1205,12 +1199,12 @@
             def load_module(self, fullname):
                 assert fullname == 'meta_path_pseudo_module'
                 # we "forget" to update sys.modules
-                return new.module('meta_path_pseudo_module')
+                return types.ModuleType('meta_path_pseudo_module')
 
-        import sys, new
+        import sys, types
         sys.meta_path.append(ImportHook())
         try:
-            import meta_path_pseudo_module
+            raises(KeyError, "import meta_path_pseudo_module")
         finally:
             sys.meta_path.pop()
 
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=3
-rev=1
+min=4
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1172,14 +1172,11 @@
         if ok:
             fdread = c_open_osfhandle(hread, 0)
             fdwrite = c_open_osfhandle(hwrite, 1)
-            if fdread == -1 or fdwrite == -1:
-                rwin32.CloseHandle(hread)
-                rwin32.CloseHandle(hwrite)
-                ok = 0
-        if not ok:
-            raise WindowsError(rwin32.GetLastError_saved(),
-                               "CreatePipe failed")
-        return (fdread, fdwrite)
+            if not (fdread == -1 or fdwrite == -1):
+                return (fdread, fdwrite)
+            rwin32.CloseHandle(pread)
+            rwin32.CloseHandle(pwrite)
+        raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
@@ -2142,6 +2139,7 @@
 eci_inheritable = eci.merge(ExternalCompilationInfo(
     separate_module_sources=[r"""
 #include <errno.h>
+#include <sys/ioctl.h>
 
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)

From pypy.commits at gmail.com  Sun Aug 28 16:16:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 Aug 2016 13:16:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-marshal3: close branch, ready to merge
Message-ID: <57c346ab.a6a5c20a.b957f.1850@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-marshal3
Changeset: r86660:b7d2cee9e28e
Date: 2016-08-28 22:15 +0200
http://bitbucket.org/pypy/pypy/changeset/b7d2cee9e28e/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Sun Aug 28 16:16:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 Aug 2016 13:16:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3.5-marshal3
Message-ID: <57c346ad.17a71c0a.aea9e.5628@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86661:4384ff755734
Date: 2016-08-28 22:16 +0200
http://bitbucket.org/pypy/pypy/changeset/4384ff755734/

Log:	hg merge py3.5-marshal3

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -837,13 +837,13 @@
             self.interned_strings.set(u, w_s1)
         return w_s1
 
-    def is_interned_str(self, s):
+    def get_interned_str(self, s):
         """Assumes an identifier (utf-8 encoded str)"""
         # interface for marshal_impl
         if not we_are_translated():
             assert type(s) is str
         u = s.decode('utf-8')
-        return self.interned_strings.get(u) is not None
+        return self.interned_strings.get(u)   # may be None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -37,7 +37,7 @@
 # different value for the highest 16 bits. Bump pypy_incremental_magic every
 # time you make pyc files incompatible
 
-pypy_incremental_magic = 64 # bump it by 16
+pypy_incremental_magic = 80 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -47,8 +47,10 @@
     def _cached_compile(space, name, source, *args):
         from rpython.config.translationoption import CACHE_DIR
         from pypy.module.marshal import interp_marshal
+        from pypy.interpreter.pycode import default_magic
 
-        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_%s' % (name,))
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_%d%s' % (
+            default_magic, name))
         try:
             if space.config.translating:
                 raise IOError("don't use the cache when translating pypy")
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -228,7 +228,7 @@
 #     CPython + 7 = default_magic  -- used by PyPy (incompatible!)
 #
 from pypy.interpreter.pycode import default_magic
-MARSHAL_VERSION_FOR_PYC = 2
+MARSHAL_VERSION_FOR_PYC = 4
 
 def get_pyc_magic(space):
     return default_magic
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -4,24 +4,30 @@
 from rpython.rlib import rstackovf
 from pypy.objspace.std.marshal_impl import marshal, get_unmarshallers
 
+#
+# Write Python objects to files and read them back.  This is primarily
+# intended for writing and reading compiled Python code, even though
+# dicts, lists, sets and frozensets, not commonly seen in code
+# objects, are supported.  Version 3 of this protocol properly
+# supports circular links and sharing.  The previous version is called
+# "2", like in Python 2.7, although it is not always compatible
+# between CPython 2.7 and CPython 3.4.  Version 4 adds small
+# optimizations in compactness.
+#
+# XXX: before py3k, there was logic to do efficiently dump()/load() on
+# a file object.  The corresponding logic is gone from CPython 3.x, so
+# I don't feel bad about killing it here too.
+#
 
-Py_MARSHAL_VERSION = 2
+Py_MARSHAL_VERSION = 4
+
 
 @unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION))
 def dump(space, w_data, w_f, w_version):
     """Write the 'data' object into the open file 'f'."""
-    # XXX: before py3k, we special-cased W_File to use a more performant
-    # FileWriter class. Should we do the same for py3k? Look also at
-    # DirectStreamWriter
-    writer = FileWriter(space, w_f)
-    try:
-        # note: bound methods are currently not supported,
-        # so we have to pass the instance in, instead.
-        ##m = Marshaller(space, writer.write, space.int_w(w_version))
-        m = Marshaller(space, writer, space.int_w(w_version))
-        m.dump_w_obj(w_data)
-    finally:
-        writer.finished()
+    # same implementation as CPython 3.x.
+    w_string = dumps(space, w_data, w_version)
+    space.call_method(w_f, 'write', w_string)
 
 @unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION))
 def dumps(space, w_data, w_version):
@@ -33,9 +39,6 @@
 
 def load(space, w_f):
     """Read one value from the file 'f' and return it."""
-    # XXX: before py3k, we special-cased W_File to use a more performant
-    # FileWriter class. Should we do the same for py3k? Look also at
-    # DirectStreamReader
     reader = FileReader(space, w_f)
     try:
         u = Unmarshaller(space, reader)
@@ -68,22 +71,6 @@
     def write(self, data):
         raise NotImplementedError("Purely abstract method")
 
-class FileWriter(AbstractReaderWriter):
-    def __init__(self, space, w_f):
-        AbstractReaderWriter.__init__(self, space)
-        try:
-            self.func = space.getattr(w_f, space.wrap('write'))
-            # XXX how to check if it is callable?
-        except OperationError as e:
-            if not e.match(space, space.w_AttributeError):
-                raise
-            raise oefmt(space.w_TypeError,
-                        "marshal.dump() 2nd arg must be file-like object")
-
-    def write(self, data):
-        space = self.space
-        space.call_function(self.func, space.newbytes(data))
-
 
 class FileReader(AbstractReaderWriter):
     def __init__(self, space, w_f):
@@ -111,33 +98,6 @@
         return ret
 
 
-class StreamReaderWriter(AbstractReaderWriter):
-    def __init__(self, space, file):
-        AbstractReaderWriter.__init__(self, space)
-        self.file = file
-        file.lock()
-
-    def finished(self):
-        self.file.unlock()
-
-class DirectStreamWriter(StreamReaderWriter):
-    """
-    XXX: this class is unused right now. Look at the comment in dump()
-    """
-    def write(self, data):
-        self.file.do_direct_write(data)
-
-class DirectStreamReader(StreamReaderWriter):
-    """
-    XXX: this class is unused right now. Look at the comment in dump()
-    """
-    def read(self, n):
-        data = self.file.direct_read(n)
-        if len(data) < n:
-            self.raise_eof()
-        return data
-
-
 class _Base(object):
     def raise_exc(self, msg):
         space = self.space
@@ -168,7 +128,15 @@
         ## self.put = putfunc
         self.writer = writer
         self.version = version
-        self.stringtable = {}
+        self.all_refs = {}
+        # all_refs = {w_obj: index} for all w_obj that are of a
+        # "reasonably sharable" type.  CPython checks the refcount of
+        # any object to know if it is sharable, independently of its
+        # type.  We can't do that.  We could do a two-pass marshaller.
+        # For now we simply add to this list all objects that marshal to
+        # more than a few fixed-sized bytes, minus ones like code
+        # objects that never appear more than once except in complete
+        # corner cases.
 
     ## currently we cannot use a put that is a bound method
     ## from outside. Same holds for get.
@@ -239,10 +207,13 @@
             rstackovf.check_stack_overflow()
             self._overflow()
 
-    def put_tuple_w(self, typecode, lst_w):
+    def put_tuple_w(self, typecode, lst_w, single_byte_size=False):
         self.start(typecode)
         lng = len(lst_w)
-        self.put_int(lng)
+        if single_byte_size:
+            self.put(chr(lng))
+        else:
+            self.put_int(lng)
         idx = 0
         while idx < lng:
             w_obj = lst_w[idx]
@@ -333,19 +304,35 @@
 
 
 def invalid_typecode(space, u, tc):
-    u.raise_exc("bad marshal data (unknown type code)")
+    u.raise_exc("bad marshal data (unknown type code %d)" % (ord(tc),))
 
 
+def _make_unmarshall_and_save_ref(func):
+    def unmarshall_save_ref(space, u, tc):
+        index = len(u.refs_w)
+        u.refs_w.append(None)
+        w_obj = func(space, u, tc)
+        u.refs_w[index] = w_obj
+        return w_obj
+    return unmarshall_save_ref
 
-class Unmarshaller(_Base):
+def _make_unmarshaller_dispatch():
     _dispatch = [invalid_typecode] * 256
     for tc, func in get_unmarshallers():
         _dispatch[ord(tc)] = func
+    for tc, func in get_unmarshallers():
+        if tc < '\x80' and _dispatch[ord(tc) + 0x80] is invalid_typecode:
+            _dispatch[ord(tc) + 0x80] = _make_unmarshall_and_save_ref(func)
+    return _dispatch
+
+
+class Unmarshaller(_Base):
+    _dispatch = _make_unmarshaller_dispatch()
 
     def __init__(self, space, reader):
         self.space = space
         self.reader = reader
-        self.stringtable_w = []
+        self.refs_w = []
 
     def get(self, n):
         assert n >= 0
@@ -355,6 +342,10 @@
         # the [0] is used to convince the annotator to return a char
         return self.get(1)[0]
 
+    def save_ref(self, typecode, w_obj):
+        if typecode >= '\x80':
+            self.refs_w.append(w_obj)
+
     def atom_str(self, typecode):
         self.start(typecode)
         lng = self.get_lng()
@@ -425,8 +416,11 @@
             self._overflow()
 
     # inlined version to save a recursion level
-    def get_tuple_w(self):
-        lng = self.get_lng()
+    def get_tuple_w(self, single_byte_size=False):
+        if single_byte_size:
+            lng = ord(self.get1())
+        else:
+            lng = self.get_lng()
         res_w = [None] * lng
         idx = 0
         space = self.space
@@ -442,9 +436,6 @@
             raise oefmt(space.w_TypeError, "NULL object in marshal data")
         return res_w
 
-    def get_list_w(self):
-        return self.get_tuple_w()[:]
-
     def _overflow(self):
         self.raise_exc('object too deeply nested to unmarshal')
 
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -199,7 +199,7 @@
     def test_bad_typecode(self):
         import marshal
         exc = raises(ValueError, marshal.loads, bytes([1]))
-        assert str(exc.value) == "bad marshal data (unknown type code)"
+        assert str(exc.value).startswith("bad marshal data (unknown type code")
 
     def test_bad_data(self):
         # If you have sufficiently little memory, the line at the end of the
diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py
--- a/pypy/module/marshal/test/test_marshalimpl.py
+++ b/pypy/module/marshal/test/test_marshalimpl.py
@@ -6,20 +6,6 @@
 class AppTestMarshalMore:
     spaceconfig = dict(usemodules=('array',))
 
-    def test_unmarshal_int64(self):
-        # test that we can unmarshal 64-bit ints on 32-bit platforms
-        # (of course we only test that if we're running on such a
-        # platform :-)
-        import marshal
-        z = marshal.loads(b'I\x00\xe4\x0bT\x02\x00\x00\x00')
-        assert z == 10000000000
-        z = marshal.loads(b'I\x00\x1c\xf4\xab\xfd\xff\xff\xff')
-        assert z == -10000000000
-        z = marshal.loads(b'I\x88\x87\x86\x85\x84\x83\x82\x01')
-        assert z == 108793946209421192
-        z = marshal.loads(b'I\xd8\xd8\xd9\xda\xdb\xdc\xcd\xfe')
-        assert z == -0x0132232425262728
-
     def test_marshal_bufferlike_object(self):
         import marshal, array
         s = marshal.dumps(array.array('b', b'asd'))
@@ -33,10 +19,6 @@
     def test_unmarshal_evil_long(self):
         import marshal
         raises(ValueError, marshal.loads, b'l\x02\x00\x00\x00\x00\x00\x00\x00')
-        z = marshal.loads(b'I\x00\xe4\x0bT\x02\x00\x00\x00')
-        assert z == 10000000000
-        z = marshal.loads(b'I\x00\x1c\xf4\xab\xfd\xff\xff\xff')
-        assert z == -10000000000
 
     def test_marshal_code_object(self):
         def foo(a, b):
@@ -49,6 +31,37 @@
             if attr_name.startswith("co_"):
                 assert getattr(code2, attr_name) == getattr(foo.__code__, attr_name)
 
+    def test_unmarshal_ascii(self):
+        import marshal
+        s = marshal.loads(b"a\x04\x00\x00\x00ab\xc2\x84")
+        assert s == "ab\xc2\x84"
+        s = marshal.loads(b"A\x04\x00\x00\x00ab\xc2\x84")
+        assert s == "ab\xc2\x84"
+        s = marshal.loads(b"z\x04ab\xc2\x84")
+        assert s == "ab\xc2\x84"
+        s = marshal.loads(b"Z\x04ab\xc2\x84")
+        assert s == "ab\xc2\x84"
+
+    def test_shared_string(self):
+        import marshal
+        x = "hello, "
+        x += "world"
+        xl = 256
+        xl **= 100
+        for version in [2, 3]:
+            s = marshal.dumps((x, x), version)
+            assert s.count(b'hello, world') == 2 if version < 3 else 1
+            y = marshal.loads(s)
+            assert y == (x, x)
+            #
+            s = marshal.dumps((xl, xl), version)
+            if version < 3:
+                assert 200 < len(s) < 250
+            else:
+                assert 100 < len(s) < 125
+            yl = marshal.loads(s)
+            assert yl == (xl, xl)
+
 
 class AppTestMarshalSmallLong(AppTestMarshalMore):
     spaceconfig = dict(usemodules=('array',),
@@ -62,6 +75,7 @@
         # NOTE: marshal is platform independent, running this test must assume
         # that self.seen gets values from the endianess of the marshal module.
         # (which is little endian!)
+        version = 2
         def __init__(self):
             self.seen = []
         def start(self, code):
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -2,6 +2,7 @@
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rstruct import ieee
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib import objectmodel
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.special import Ellipsis
@@ -29,14 +30,14 @@
 TYPE_STOPITER  = 'S'
 TYPE_ELLIPSIS  = '.'
 TYPE_INT       = 'i'
-TYPE_INT64     = 'I'
 TYPE_FLOAT     = 'f'
 TYPE_BINARY_FLOAT = 'g'
 TYPE_COMPLEX   = 'x'
 TYPE_BINARY_COMPLEX = 'y'
 TYPE_LONG      = 'l'
-TYPE_STRING    = 's'
-TYPE_STRINGREF = 'R'
+TYPE_STRING    = 's'     # a *byte* string, not unicode
+TYPE_INTERNED  = 't'
+TYPE_REF       = 'r'
 TYPE_TUPLE     = '('
 TYPE_LIST      = '['
 TYPE_DICT      = '{'
@@ -45,6 +46,15 @@
 TYPE_UNKNOWN   = '?'
 TYPE_SET       = '<'
 TYPE_FROZENSET = '>'
+FLAG_REF       = 0x80    # bit added to mean "add obj to index"
+FLAG_DONE      = '\x00'
+
+# the following typecodes have been added in version 4.
+TYPE_ASCII                = 'a'   # never generated so far by pypy
+TYPE_ASCII_INTERNED       = 'A'   # never generated so far by pypy
+TYPE_SMALL_TUPLE          = ')'
+TYPE_SHORT_ASCII          = 'z'   # never generated so far by pypy
+TYPE_SHORT_ASCII_INTERNED = 'Z'   # never generated so far by pypy
 
 
 _marshallers = []
@@ -56,12 +66,33 @@
         return f
     return _decorator
 
-def unmarshaller(tc):
+def unmarshaller(tc, save_ref=False):
     def _decorator(f):
+        assert tc < '\x80'
         _unmarshallers.append((tc, f))
+        if save_ref:
+            tcref = chr(ord(tc) + 0x80)
+            _unmarshallers.append((tcref, f))
         return f
     return _decorator
 
+def write_ref(typecode, w_obj, m):
+    if m.version < 3:
+        return typecode     # not writing object references
+    try:
+        index = m.all_refs[w_obj]
+    except KeyError:
+        # we don't support long indices
+        index = len(m.all_refs)
+        if index >= 0x7fffffff:
+            return typecode
+        m.all_refs[w_obj] = index
+        return chr(ord(typecode) + FLAG_REF)
+    else:
+        # write the reference index to the stream
+        m.atom_int(TYPE_REF, index)
+        return FLAG_DONE
+
 def marshal(space, w_obj, m):
     # _marshallers_unroll is defined at the end of the file
     # NOTE that if w_obj is a heap type, like an instance of a
@@ -80,7 +111,9 @@
         if e.match(space, space.w_TypeError):
             raise oefmt(space.w_ValueError, "unmarshallable object")
         raise
-    m.atom_str(TYPE_STRING, s.as_str())
+    typecode = write_ref(TYPE_STRING, w_obj, m)
+    if typecode != FLAG_DONE:
+        m.atom_str(typecode, s.as_str())
 
 def get_unmarshallers():
     return _unmarshallers
@@ -130,37 +163,27 @@
 
 @marshaller(W_IntObject)
 def marshal_int(space, w_int, m):
-    if LONG_BIT == 32:
+    y = w_int.intval >> 31
+    if y and y != -1:
+        marshal_long(space, w_int, m)
+    else:
         m.atom_int(TYPE_INT, w_int.intval)
-    else:
-        y = w_int.intval >> 31
-        if y and y != -1:
-            m.atom_int64(TYPE_INT64, w_int.intval)
-        else:
-            m.atom_int(TYPE_INT, w_int.intval)
 
 @unmarshaller(TYPE_INT)
 def unmarshal_int(space, u, tc):
     return space.newint(u.get_int())
 
- at unmarshaller(TYPE_INT64)
-def unmarshal_int64(space, u, tc):
-    lo = u.get_int()    # get the first 32 bits
-    hi = u.get_int()    # get the next 32 bits
-    if LONG_BIT >= 64:
-        x = (hi << 32) | (lo & (2**32-1))    # result fits in an int
-    else:
-        x = (r_longlong(hi) << 32) | r_longlong(r_uint(lo))  # get a r_longlong
-    return space.wrap(x)
-
 
 @marshaller(W_AbstractLongObject)
 def marshal_long(space, w_long, m):
     from rpython.rlib.rarithmetic import r_ulonglong
-    m.start(TYPE_LONG)
+    typecode = write_ref(TYPE_LONG, w_long, m)
+    if typecode == FLAG_DONE:
+        return
+    m.start(typecode)
     SHIFT = 15
     MASK = (1 << SHIFT) - 1
-    num = w_long.asbigint()
+    num = space.bigint_w(w_long)
     sign = num.sign
     num = num.abs()
     total_length = (num.bit_length() + (SHIFT - 1)) / SHIFT
@@ -248,59 +271,79 @@
 
 @marshaller(W_BytesObject)
 def marshal_bytes(space, w_str, m):
-    s = w_str.unwrap(space)
-    m.atom_str(TYPE_STRING, s)
+    typecode = write_ref(TYPE_STRING, w_str, m)
+    if typecode != FLAG_DONE:
+        s = space.bytes_w(w_str)
+        m.atom_str(typecode, s)
 
 @unmarshaller(TYPE_STRING)
 def unmarshal_bytes(space, u, tc):
     return space.newbytes(u.get_str())
 
- at unmarshaller(TYPE_STRINGREF)
-def unmarshal_stringref(space, u, tc):
-    idx = u.get_int()
-    try:
-        return u.stringtable_w[idx]
-    except IndexError:
-        raise oefmt(space.w_ValueError, "bad marshal data")
 
+def _marshal_tuple(space, tuple_w, m):
+    if m.version >= 4 and len(tuple_w) < 256:
+        typecode = TYPE_SMALL_TUPLE
+        single_byte_size = True
+    else:
+        typecode = TYPE_TUPLE
+        single_byte_size = False
+    # -- does it make any sense to try to share tuples, based on the
+    # -- *identity* of the tuple object?  I'd guess not really
+    #typecode = write_ref(typecode, w_tuple, m)
+    #if typecode != FLAG_DONE:
+    m.put_tuple_w(typecode, tuple_w, single_byte_size=single_byte_size)
 
 @marshaller(W_AbstractTupleObject)
 def marshal_tuple(space, w_tuple, m):
-    items = w_tuple.tolist()
-    m.put_tuple_w(TYPE_TUPLE, items)
+    _marshal_tuple(space, w_tuple.tolist(), m)
 
 @unmarshaller(TYPE_TUPLE)
 def unmarshal_tuple(space, u, tc):
     items_w = u.get_tuple_w()
     return space.newtuple(items_w)
 
+ at unmarshaller(TYPE_SMALL_TUPLE)
+def unmarshal_tuple(space, u, tc):
+    items_w = u.get_tuple_w(single_byte_size=True)
+    return space.newtuple(items_w)
+
 
 @marshaller(W_ListObject)
 def marshal_list(space, w_list, m):
-    items = w_list.getitems()[:]
-    m.put_tuple_w(TYPE_LIST, items)
+    typecode = write_ref(TYPE_LIST, w_list, m)
+    if typecode != FLAG_DONE:
+        items = w_list.getitems()[:]
+        m.put_tuple_w(typecode, items)
 
- at unmarshaller(TYPE_LIST)
+ at unmarshaller(TYPE_LIST, save_ref=True)
 def unmarshal_list(space, u, tc):
-    items_w = u.get_list_w()
-    return space.newlist(items_w)
+    w_obj = space.newlist([])
+    u.save_ref(tc, w_obj)
+    for w_item in u.get_tuple_w():
+        w_obj.append(w_item)
+    return w_obj
 
 
 @marshaller(W_DictMultiObject)
 def marshal_dict(space, w_dict, m):
-    m.start(TYPE_DICT)
+    typecode = write_ref(TYPE_DICT, w_dict, m)
+    if typecode == FLAG_DONE:
+        return
+    m.start(typecode)
     for w_tuple in w_dict.items():
         w_key, w_value = space.fixedview(w_tuple, 2)
         m.put_w_obj(w_key)
         m.put_w_obj(w_value)
     m.atom(TYPE_NULL)
 
- at unmarshaller(TYPE_DICT)
+ at unmarshaller(TYPE_DICT, save_ref=True)
 def unmarshal_dict(space, u, tc):
     # since primitive lists are not optimized and we don't know
     # the dict size in advance, use the dict's setitem instead
     # of building a list of tuples.
     w_dic = space.newdict()
+    u.save_ref(tc, w_dic)
     while 1:
         w_key = u.get_w_obj(allow_null=True)
         if w_key is None:
@@ -314,14 +357,9 @@
     return None
 
 
-def _put_str_list(space, m, strlist):
-    m.atom_int(TYPE_TUPLE, len(strlist))
-    atom_str = m.atom_str
-    for item in strlist:
-        atom_str(TYPE_STRING, item)
-
 @marshaller(PyCode)
 def marshal_pycode(space, w_pycode, m):
+    # (no attempt at using write_ref here, there is little point imho)
     m.start(TYPE_CODE)
     # see pypy.interpreter.pycode for the layout
     x = space.interp_w(PyCode, w_pycode)
@@ -331,105 +369,161 @@
     m.put_int(x.co_stacksize)
     m.put_int(x.co_flags)
     m.atom_str(TYPE_STRING, x.co_code)
-    m.put_tuple_w(TYPE_TUPLE, x.co_consts_w)
-    _put_str_list(space, m, [space.str_w(w_name) for w_name in x.co_names_w])
-    _put_str_list(space, m, x.co_varnames)
-    _put_str_list(space, m, x.co_freevars)
-    _put_str_list(space, m, x.co_cellvars)
-    m.atom_str(TYPE_STRING, x.co_filename)
-    m.atom_str(TYPE_STRING, x.co_name)
+    _marshal_tuple(space, x.co_consts_w, m)
+    _marshal_tuple(space, x.co_names_w, m)   # list of w_unicodes
+    co_varnames_w = [space.wrap(s.decode('utf-8')) for s in x.co_varnames]
+    co_freevars_w = [space.wrap(s.decode('utf-8')) for s in x.co_freevars]
+    co_cellvars_w = [space.wrap(s.decode('utf-8')) for s in x.co_cellvars]
+    _marshal_tuple(space, co_varnames_w, m)  # more lists, now of w_unicodes
+    _marshal_tuple(space, co_freevars_w, m)
+    _marshal_tuple(space, co_cellvars_w, m)
+    _marshal_unicode(space, x.co_filename, m)
+    _marshal_unicode(space, x.co_name, m)
     m.put_int(x.co_firstlineno)
     m.atom_str(TYPE_STRING, x.co_lnotab)
 
 # helper for unmarshalling "tuple of string" objects
 # into rpython-level lists of strings.  Only for code objects.
 
-def unmarshal_str(u):
+def _unmarshal_strlist(u):
+    items_w = _unmarshal_tuple_w(u)
+    return [u.space.unicode_w(w_item).encode('utf-8') for w_item in items_w]
+
+def _unmarshal_tuple_w(u):
     w_obj = u.get_w_obj()
     try:
-        return u.space.bytes_w(w_obj)
-    except OperationError as e:
-        if e.match(u.space, u.space.w_TypeError):
-            u.raise_exc('invalid marshal data for code object')
-        else:
-            raise
-
-def unmarshal_str0(u):
-    w_obj = u.get_w_obj()
-    try:
-        return u.space.bytes0_w(w_obj)
+        return u.space.fixedview(w_obj)
     except OperationError as e:
         if e.match(u.space, u.space.w_TypeError):
             u.raise_exc('invalid marshal data for code object')
         raise
 
-def unmarshal_strlist(u, tc):
-    lng = u.atom_lng(tc)
-    return [unmarshal_str(u) for i in range(lng)]
-
- at unmarshaller(TYPE_CODE)
+ at unmarshaller(TYPE_CODE, save_ref=True)
 def unmarshal_pycode(space, u, tc):
+    w_codeobj = objectmodel.instantiate(PyCode)
+    u.save_ref(tc, w_codeobj)
     argcount    = u.get_int()
     kwonlyargcount = u.get_int()
     nlocals     = u.get_int()
     stacksize   = u.get_int()
     flags       = u.get_int()
-    code        = unmarshal_str(u)
-    u.start(TYPE_TUPLE)
-    consts_w    = u.get_tuple_w()
-    # copy in order not to merge it with anything else
-    names       = unmarshal_strlist(u, TYPE_TUPLE)
-    varnames    = unmarshal_strlist(u, TYPE_TUPLE)
-    freevars    = unmarshal_strlist(u, TYPE_TUPLE)
-    cellvars    = unmarshal_strlist(u, TYPE_TUPLE)
-    filename    = unmarshal_str0(u)
-    name        = unmarshal_str(u)
+    code        = space.bytes_w(u.get_w_obj())
+    consts_w    = _unmarshal_tuple_w(u)   
+    names       = _unmarshal_strlist(u)
+    varnames    = _unmarshal_strlist(u)
+    freevars    = _unmarshal_strlist(u)
+    cellvars    = _unmarshal_strlist(u)
+    filename    = space.unicode0_w(u.get_w_obj()).encode('utf-8')
+    name        = space.unicode_w(u.get_w_obj()).encode('utf-8')
     firstlineno = u.get_int()
-    lnotab      = unmarshal_str(u)
-    return PyCode(space, argcount, kwonlyargcount, nlocals, stacksize, flags,
+    lnotab      = space.bytes_w(u.get_w_obj())
+    PyCode.__init__(w_codeobj,
+                  space, argcount, kwonlyargcount, nlocals, stacksize, flags,
                   code, consts_w[:], names, varnames, filename,
                   name, firstlineno, lnotab, freevars, cellvars)
+    return w_codeobj
 
 
+def _marshal_unicode(space, s, m, w_unicode=None):
+    if m.version >= 3:
+        w_interned = space.get_interned_str(s)
+    else:
+        w_interned = None
+    if w_interned is not None:
+        w_unicode = w_interned    # use the interned W_UnicodeObject
+        typecode = TYPE_INTERNED  #   as a key for u.all_refs
+    else:
+        typecode = TYPE_UNICODE
+    if w_unicode is not None:
+        typecode = write_ref(typecode, w_unicode, m)
+    if typecode != FLAG_DONE:
+        m.atom_str(typecode, s)
+
 @marshaller(W_UnicodeObject)
 def marshal_unicode(space, w_unicode, m):
     s = unicodehelper.encode_utf8(space, space.unicode_w(w_unicode),
                                   allow_surrogates=True)
-    m.atom_str(TYPE_UNICODE, s)
+    _marshal_unicode(space, s, m, w_unicode=w_unicode)
 
 @unmarshaller(TYPE_UNICODE)
 def unmarshal_unicode(space, u, tc):
-    return space.wrap(unicodehelper.decode_utf8(space, u.get_str(),
-                                                allow_surrogates=True))
+    uc = unicodehelper.decode_utf8(space, u.get_str(), allow_surrogates=True)
+    return space.newunicode(uc)
+
+ at unmarshaller(TYPE_INTERNED)
+def unmarshal_bytes(space, u, tc):
+    return space.new_interned_str(u.get_str())
+
+def _unmarshal_ascii(u, short_length, interned):
+    if short_length:
+        lng = ord(u.get1())
+    else:
+        lng = u.get_lng()
+    s = u.get(lng)
+    w_u = u.space.newunicode(s.decode('latin-1'))
+    if interned:
+        w_u = u.space.new_interned_w_str(w_u)
+    return w_u
+
+ at unmarshaller(TYPE_ASCII)    # nb. never generated by pypy so far
+def unmarshal_ascii(space, u, tc):
+    return _unmarshal_ascii(u, False, False)
+ at unmarshaller(TYPE_ASCII_INTERNED)
+def unmarshal_ascii(space, u, tc):
+    return _unmarshal_ascii(u, False, True)
+ at unmarshaller(TYPE_SHORT_ASCII)
+def unmarshal_ascii(space, u, tc):
+    return _unmarshal_ascii(u, True, False)
+ at unmarshaller(TYPE_SHORT_ASCII_INTERNED)
+def unmarshal_ascii(space, u, tc):
+    return _unmarshal_ascii(u, True, True)
+
 
 @marshaller(W_SetObject)
 def marshal_set(space, w_set, m):
-    lis_w = space.fixedview(w_set)
-    m.put_tuple_w(TYPE_SET, lis_w)
+    typecode = write_ref(TYPE_SET, w_set, m)
+    if typecode != FLAG_DONE:
+        lis_w = space.fixedview(w_set)
+        m.put_tuple_w(typecode, lis_w)
 
- at unmarshaller(TYPE_SET)
+ at unmarshaller(TYPE_SET, save_ref=True)
 def unmarshal_set(space, u, tc):
-    return unmarshal_set_frozenset(space, u, tc)
+    w_set = space.call_function(space.w_set)
+    u.save_ref(tc, w_set)
+    _unmarshal_set_frozenset(space, u, w_set)
+    return w_set
 
 
 @marshaller(W_FrozensetObject)
 def marshal_frozenset(space, w_frozenset, m):
-    lis_w = space.fixedview(w_frozenset)
-    m.put_tuple_w(TYPE_FROZENSET, lis_w)
+    typecode = write_ref(TYPE_FROZENSET, w_frozenset, m)
+    if typecode != FLAG_DONE:
+        lis_w = space.fixedview(w_frozenset)
+        m.put_tuple_w(typecode, lis_w)
 
-def unmarshal_set_frozenset(space, u, tc):
+def _unmarshal_set_frozenset(space, u, w_set):
     lng = u.get_lng()
-    w_set = space.call_function(space.w_set)
     for i in xrange(lng):
         w_obj = u.get_w_obj()
         space.call_method(w_set, "add", w_obj)
-    if tc == TYPE_FROZENSET:
-        w_set = space.call_function(space.w_frozenset, w_set)
-    return w_set
 
 @unmarshaller(TYPE_FROZENSET)
 def unmarshal_frozenset(space, u, tc):
-    return unmarshal_set_frozenset(space, u, tc)
+    w_set = space.call_function(space.w_set)
+    _unmarshal_set_frozenset(space, u, w_set)
+    return space.call_function(space.w_frozenset, w_set)
+
+
+ at unmarshaller(TYPE_REF)
+def unmarshal_ref(space, u, tc):
+    index = u.get_lng()
+    if 0 <= index < len(u.refs_w):
+        w_obj = u.refs_w[index]
+    else:
+        w_obj = None
+    if w_obj is None:
+        raise oefmt(space.w_ValueError, "bad marshal data (invalid reference)")
+    return w_obj
 
 
 _marshallers_unroll = unrolling_iterable(_marshallers)

From pypy.commits at gmail.com  Sun Aug 28 16:17:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 Aug 2016 13:17:29 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: done
Message-ID: <57c346d9.a427c20a.17051.1bff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5687:15bcc24aeee2
Date: 2016-08-28 22:17 +0200
http://bitbucket.org/pypy/extradoc/changeset/15bcc24aeee2/

Log:	done

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -11,8 +11,6 @@
   We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
   Matti's opinion on that would be great!
 
-* arigo: New marshal format
-
 
 Finished
 --------
@@ -72,6 +70,8 @@
   (POSIX-DONE, missing Win32)
 
 * The marshal format has been made more compact and efficient
+  (DONE, maybe a small optimization left---TYPE_*ASCII*---that
+  depends on compact unicode representation)
 
 * enum: Support for enumeration types (PEP 435).
 

From pypy.commits at gmail.com  Sun Aug 28 16:25:26 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sun, 28 Aug 2016 13:25:26 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update QCGC codebase
Message-ID: <57c348b6.d42f1c0a.d3e46.5930@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86662:a91d70d81b8d
Date: 2016-08-28 22:23 +0200
http://bitbucket.org/pypy/pypy/changeset/a91d70d81b8d/

Log:	Update QCGC codebase

diff --git a/rpython/translator/c/src/qcgc/arena.c b/rpython/translator/c/src/qcgc/arena.c
--- a/rpython/translator/c/src/qcgc/arena.c
+++ b/rpython/translator/c/src/qcgc/arena.c
@@ -162,9 +162,11 @@
 bool qcgc_arena_sweep(arena_t *arena) {
 #if CHECKED
 	assert(arena != NULL);
+	assert(qcgc_arena_is_coalesced(arena));
 #endif
 	bool free = true;
 	bool coalesce = false;
+	bool add_to_free_list = false;
 	size_t last_free_cell = QCGC_ARENA_FIRST_CELL_INDEX;
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
@@ -188,22 +190,27 @@
 					last_free_cell = cell;
 				}
 				coalesce = true;
+				add_to_free_list = true;
 				break;
 			case BLOCK_BLACK:
 				set_blocktype(arena, cell, BLOCK_WHITE);
-				if (coalesce) {
+				if (add_to_free_list) {
 					qcgc_fit_allocator_add(&(arena->cells[last_free_cell]),
 							cell - last_free_cell);
 				}
 				free = false;
 				coalesce = false;
+				add_to_free_list = false;
 				break;
 		}
 	}
-	if (coalesce && !free) {
+	if (add_to_free_list && !free) {
 		qcgc_fit_allocator_add(&(arena->cells[last_free_cell]),
 							QCGC_ARENA_CELLS_COUNT - last_free_cell);
 	}
+#if CHECKED
+	assert(qcgc_arena_is_coalesced(arena));
+#endif
 	return free;
 }
 
diff --git a/rpython/translator/c/src/qcgc/object.h b/rpython/translator/c/src/qcgc/object.h
--- a/rpython/translator/c/src/qcgc/object.h
+++ b/rpython/translator/c/src/qcgc/object.h
@@ -4,6 +4,10 @@
 #include "config.h"
 #include <stdint.h>
 
+/**
+ * The lower half of flags is reserved for the library, the upper half for
+ * clients
+ */
 #define QCGC_GRAY_FLAG (1<<0)
 #define QCGC_PREBUILT_OBJECT (1<<1)
 #define QCGC_PREBUILT_REGISTERED (1<<2)

From pypy.commits at gmail.com  Sun Aug 28 17:49:34 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 28 Aug 2016 14:49:34 -0700 (PDT)
Subject: [pypy-commit] pypy rpython-resync: Close branch rpython-resync
Message-ID: <57c35c6e.28eac20a.93a56.39f0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rpython-resync
Changeset: r86663:d3ea050834b6
Date: 2016-08-28 22:49 +0100
http://bitbucket.org/pypy/pypy/changeset/d3ea050834b6/

Log:	Close branch rpython-resync


From pypy.commits at gmail.com  Sun Aug 28 17:49:53 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 28 Aug 2016 14:49:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged rpython-resync into default
Message-ID: <57c35c81.c62f1c0a.57e03.73c2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86664:6b0c10d6e524
Date: 2016-08-28 22:49 +0100
http://bitbucket.org/pypy/pypy/changeset/6b0c10d6e524/

Log:	Merged rpython-resync into default

	Backport rpython changes from py3k and py3.5.

diff too long, truncating to 2000 out of 208811 lines

diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20031017)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -64,7 +64,7 @@
         AsyncAction.__init__(self, space)
         self.pending_signal = -1
         self.fire_in_another_thread = False
-        #
+
         @rgc.no_collect
         def _after_thread_switch():
             if self.fire_in_another_thread:
@@ -251,7 +251,7 @@
         except OSError as e:
             if e.errno == errno.EBADF:
                 raise oefmt(space.w_ValueError, "invalid fd")
-    old_fd = pypysig_set_wakeup_fd(fd)
+    old_fd = pypysig_set_wakeup_fd(fd, True)
     return space.wrap(intmask(old_fd))
 
 
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -176,6 +176,7 @@
 
 
 SOCK_DGRAM SOCK_RAW SOCK_RDM SOCK_SEQPACKET SOCK_STREAM
+SOCK_CLOEXEC
 
 SOL_SOCKET SOL_IPX SOL_AX25 SOL_ATALK SOL_NETROM SOL_ROSE
 
@@ -319,6 +320,8 @@
                                           [('p_proto', rffi.INT),
                                            ])
 
+CConfig.HAVE_ACCEPT4 = platform.Has('accept4')
+
 if _POSIX:
     CConfig.nfds_t = platform.SimpleType('nfds_t')
     CConfig.pollfd = platform.Struct('struct pollfd',
@@ -541,6 +544,12 @@
 socketaccept = external('accept', [socketfd_type, sockaddr_ptr,
                                    socklen_t_ptr], socketfd_type,
                         save_err=SAVE_ERR)
+HAVE_ACCEPT4 = cConfig.HAVE_ACCEPT4
+if HAVE_ACCEPT4:
+    socketaccept4 = external('accept4', [socketfd_type, sockaddr_ptr,
+                                         socklen_t_ptr, rffi.INT],
+                                        socketfd_type,
+                             save_err=SAVE_ERR)
 socketbind = external('bind', [socketfd_type, sockaddr_ptr, socklen_t],
                               rffi.INT, save_err=SAVE_ERR)
 socketlisten = external('listen', [socketfd_type, rffi.INT], rffi.INT,
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -10,6 +10,7 @@
     _immutable_ = True
 
     def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
         raise NotImplementedError
 
     def __len__(self):
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1222,6 +1222,9 @@
         # base is supposed to be positive or 0.0, which means we use e
         if base == 10.0:
             return _loghelper(math.log10, self)
+        if base == 2.0:
+            from rpython.rlib import rfloat
+            return _loghelper(rfloat.log2, self)
         ret = _loghelper(math.log, self)
         if base != 0.0:
             ret /= math.log(base)
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -126,6 +126,10 @@
         "SSL_OP_NO_COMPRESSION")
     SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger(
         "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS")
+    SSL_OP_CIPHER_SERVER_PREFERENCE = rffi_platform.ConstantInteger(
+        "SSL_OP_CIPHER_SERVER_PREFERENCE")
+    SSL_OP_SINGLE_DH_USE = rffi_platform.ConstantInteger(
+        "SSL_OP_SINGLE_DH_USE")
     HAS_SNI = rffi_platform.Defined("SSL_CTRL_SET_TLSEXT_HOSTNAME")
     HAS_NPN = rffi_platform.Defined("OPENSSL_NPN_NEGOTIATED")
     SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE")
@@ -307,6 +311,8 @@
 
 if HAVE_OPENSSL_RAND:
     ssl_external('RAND_add', [rffi.CCHARP, rffi.INT, rffi.DOUBLE], lltype.Void)
+    ssl_external('RAND_bytes', [rffi.UCHARP, rffi.INT], rffi.INT)
+    ssl_external('RAND_pseudo_bytes', [rffi.UCHARP, rffi.INT], rffi.INT)
     ssl_external('RAND_status', [], rffi.INT)
     if HAVE_OPENSSL_RAND_EGD:
         ssl_external('RAND_egd', [rffi.CCHARP], rffi.INT)
@@ -465,6 +471,7 @@
 ssl_external('GENERAL_NAME_print', [BIO, GENERAL_NAME], rffi.INT)
 ssl_external('pypy_GENERAL_NAME_dirn', [GENERAL_NAME], X509_NAME,
              macro=True)
+
 ssl_external('pypy_GENERAL_NAME_uri', [GENERAL_NAME], ASN1_IA5STRING,
              macro=True)
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -223,7 +223,7 @@
             pass
 
 if _WIN32:
-    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h']
+    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h']
     libraries = []
 else:
     if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')):
@@ -254,10 +254,11 @@
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
+    CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
-        CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
+        TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),
@@ -265,6 +266,12 @@
                            ('tms_cutime', rffi.INT),
                            ('tms_cstime', rffi.INT)])
 
+        WINSIZE = rffi_platform.Struct(
+            'struct winsize', [('ws_row', rffi.USHORT),
+                           ('ws_col', rffi.USHORT),
+                           ('ws_xpixel', rffi.USHORT),
+                           ('ws_ypixel', rffi.USHORT)])
+
     GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
     SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")
 
@@ -365,15 +372,27 @@
         raise OSError(get_saved_errno(), '%s failed' % name)
     return result
 
+def _dup(fd, inheritable=True):
+    validate_fd(fd)
+    if inheritable:
+        res = c_dup(fd)
+    else:
+        res = c_dup_noninheritable(fd)
+    return res
+
 @replace_os_function('dup')
-def dup(fd):
-    validate_fd(fd)
-    return handle_posix_error('dup', c_dup(fd))
+def dup(fd, inheritable=True):
+    res = _dup(fd, inheritable)
+    return handle_posix_error('dup', res)
 
 @replace_os_function('dup2')
-def dup2(fd, newfd):
+def dup2(fd, newfd, inheritable=True):
     validate_fd(fd)
-    handle_posix_error('dup2', c_dup2(fd, newfd))
+    if inheritable:
+        res = c_dup2(fd, newfd)
+    else:
+        res = c_dup2_noninheritable(fd, newfd)
+    handle_posix_error('dup2', res)
 
 #___________________________________________________________________
 
@@ -628,6 +647,8 @@
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
     c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+    c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 else:
     dirent_config = {}
 
@@ -1113,37 +1134,77 @@
     c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
                                                     rffi.INT],
                                 rffi.INT)
+    HAVE_PIPE2 = False
+    HAVE_DUP3 = False
+    O_CLOEXEC = None
 else:
     INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
     c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT,
                       save_err=rffi.RFFI_SAVE_ERRNO)
+    class CConfig:
+        _compilation_info_ = eci
+        HAVE_PIPE2 = rffi_platform.Has('pipe2')
+        HAVE_DUP3 = rffi_platform.Has('dup3')
+        O_CLOEXEC = rffi_platform.DefinedConstantInteger('O_CLOEXEC')
+    config = rffi_platform.configure(CConfig)
+    HAVE_PIPE2 = config['HAVE_PIPE2']
+    HAVE_DUP3 = config['HAVE_DUP3']
+    O_CLOEXEC = config['O_CLOEXEC']
+    if HAVE_PIPE2:
+        c_pipe2 = external('pipe2', [INT_ARRAY_P, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('pipe')
-def pipe():
+def pipe(flags=0):
+    # 'flags' might be ignored.  Check the result.
     if _WIN32:
+        # 'flags' ignored
         pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
         try:
-            if not CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                raise WindowsError(rwin32.GetLastError_saved(),
-                                   "CreatePipe failed")
+            ok = CreatePipe(
+                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0)
             hread = rffi.cast(rffi.INTPTR_T, pread[0])
             hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
         finally:
             lltype.free(pwrite, flavor='raw')
             lltype.free(pread, flavor='raw')
-        fdread = c_open_osfhandle(hread, 0)
-        fdwrite = c_open_osfhandle(hwrite, 1)
+        if ok:
+            fdread = c_open_osfhandle(hread, 0)
+            fdwrite = c_open_osfhandle(hwrite, 1)
+            if fdread == -1 or fdwrite == -1:
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
+                ok = 0
+        if not ok:
+            raise WindowsError(rwin32.GetLastError_saved(),
+                               "CreatePipe failed")
         return (fdread, fdwrite)
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
-            handle_posix_error('pipe', c_pipe(filedes))
+            if HAVE_PIPE2 and _pipe2_syscall.attempt_syscall():
+                res = c_pipe2(filedes, flags)
+                if _pipe2_syscall.fallback(res):
+                    res = c_pipe(filedes)
+            else:
+                res = c_pipe(filedes)      # 'flags' ignored
+            handle_posix_error('pipe', res)
             return (widen(filedes[0]), widen(filedes[1]))
         finally:
             lltype.free(filedes, flavor='raw')
 
+def pipe2(flags):
+    # Only available if there is really a c_pipe2 function.
+    # No fallback to pipe() if we get ENOSYS.
+    filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
+    try:
+        res = c_pipe2(filedes, flags)
+        handle_posix_error('pipe2', res)
+        return (widen(filedes[0]), widen(filedes[1]))
+    finally:
+        lltype.free(filedes, flavor='raw')
+
 c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
                   save_err=rffi.RFFI_SAVE_ERRNO,)
 c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
@@ -2079,14 +2140,46 @@
 
 
 eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=["""
+    separate_module_sources=[r"""
+#include <errno.h>
+
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)
 {
-    /* XXX minimal impl. XXX */
-    int request = inheritable ? FIONCLEX : FIOCLEX;
-    return ioctl(fd, request, NULL);
+    static int ioctl_works = -1;
+    int flags;
+
+    if (ioctl_works != 0) {
+        int request = inheritable ? FIONCLEX : FIOCLEX;
+        int err = ioctl(fd, request, NULL);
+        if (!err) {
+            ioctl_works = 1;
+            return 0;
+        }
+
+        if (errno != ENOTTY && errno != EACCES) {
+            return -1;
+        }
+        else {
+            /* ENOTTY: The ioctl is declared but not supported by the
+               kernel.  EACCES: SELinux policy, this can be the case on
+               Android. */
+            ioctl_works = 0;
+        }
+        /* fallback to fcntl() if ioctl() does not work */
+    }
+
+    flags = fcntl(fd, F_GETFD);
+    if (flags < 0)
+        return -1;
+
+    if (inheritable)
+        flags &= ~FD_CLOEXEC;
+    else
+        flags |= FD_CLOEXEC;
+    return fcntl(fd, F_SETFD, flags);
 }
+
 RPY_EXTERN
 int rpy_get_inheritable(int fd)
 {
@@ -2095,8 +2188,64 @@
         return -1;
     return !(flags & FD_CLOEXEC);
 }
-    """],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+RPY_EXTERN
+int rpy_dup_noninheritable(int fd)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUPFD_CLOEXEC
+    return fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+    fd = dup(fd);
+    if (fd >= 0) {
+        if (rpy_set_inheritable(fd, 0) != 0) {
+            close(fd);
+            return -1;
+        }
+    }
+    return fd;
+#endif
+}
+
+RPY_EXTERN
+int rpy_dup2_noninheritable(int fd, int fd2)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUP2FD_CLOEXEC
+    return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
+
+#else
+# if %(HAVE_DUP3)d   /* HAVE_DUP3 */
+    static int dup3_works = -1;
+    if (dup3_works != 0) {
+        if (dup3(fd, fd2, O_CLOEXEC) >= 0)
+            return 0;
+        if (dup3_works == -1)
+            dup3_works = (errno != ENOSYS);
+        if (dup3_works)
+            return -1;
+    }
+# endif
+    if (dup2(fd, fd2) < 0)
+        return -1;
+    if (rpy_set_inheritable(fd2, 0) != 0) {
+        close(fd2);
+        return -1;
+    }
+    return 0;
+#endif
+}
+    """ % {'HAVE_DUP3': HAVE_DUP3}],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
 
 c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
@@ -2104,12 +2253,56 @@
 c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
                              compilation_info=eci_inheritable)
+c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                compilation_info=eci_inheritable)
+c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                 compilation_info=eci_inheritable)
 
 def set_inheritable(fd, inheritable):
-    error = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', error)
+    result = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', result)
 
 def get_inheritable(fd):
     res = c_get_inheritable(fd)
     res = handle_posix_error('get_inheritable', res)
     return res != 0
+
+class SetNonInheritableCache(object):
+    """Make one prebuilt instance of this for each path that creates
+    file descriptors, where you don't necessarily know if that function
+    returns inheritable or non-inheritable file descriptors.
+    """
+    _immutable_fields_ = ['cached_inheritable?']
+    cached_inheritable = -1    # -1 = don't know yet; 0 = off; 1 = on
+
+    def set_non_inheritable(self, fd):
+        if self.cached_inheritable == -1:
+            self.cached_inheritable = get_inheritable(fd)
+        if self.cached_inheritable == 1:
+            # 'fd' is inheritable; we must manually turn it off
+            set_inheritable(fd, False)
+
+    def _cleanup_(self):
+        self.cached_inheritable = -1
+
+class ENoSysCache(object):
+    """Cache whether a system call returns ENOSYS or not."""
+    _immutable_fields_ = ['cached_nosys?']
+    cached_nosys = -1      # -1 = don't know; 0 = no; 1 = yes, getting ENOSYS
+
+    def attempt_syscall(self):
+        return self.cached_nosys != 1
+
+    def fallback(self, res):
+        nosys = self.cached_nosys
+        if nosys == -1:
+            nosys = (res < 0 and get_saved_errno() == errno.ENOSYS)
+            self.cached_nosys = nosys
+        return nosys
+
+    def _cleanup_(self):
+        self.cached_nosys = -1
+
+_pipe2_syscall = ENoSysCache()
diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py
--- a/rpython/rlib/rsignal.py
+++ b/rpython/rlib/rsignal.py
@@ -31,7 +31,7 @@
     signal_names.append('CTRL_BREAK_EVENT')
     CTRL_C_EVENT = 0
     CTRL_BREAK_EVENT = 1
-includes = ['stdlib.h', 'src/signals.h']
+includes = ['stdlib.h', 'src/signals.h', 'signal.h']
 if sys.platform != 'win32':
     includes.append('sys/time.h')
 
@@ -47,7 +47,9 @@
     _compilation_info_ = eci
 
 if sys.platform != 'win32':
-    for name in """ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF""".split():
+    for name in """
+        ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF
+        SIG_BLOCK SIG_UNBLOCK SIG_SETMASK""".split():
         setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name))
 
     CConfig.timeval = rffi_platform.Struct(
@@ -71,7 +73,8 @@
 pypysig_default = external('pypysig_default', [rffi.INT], lltype.Void)
 pypysig_setflag = external('pypysig_setflag', [rffi.INT], lltype.Void)
 pypysig_reinstall = external('pypysig_reinstall', [rffi.INT], lltype.Void)
-pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd', [rffi.INT], rffi.INT)
+pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd',
+                                 [rffi.INT, rffi.INT], rffi.INT)
 pypysig_poll = external('pypysig_poll', [], rffi.INT, releasegil=False)
 # don't bother releasing the GIL around a call to pypysig_poll: it's
 # pointless and a performance issue
@@ -98,3 +101,20 @@
                            [rffi.INT, itimervalP, itimervalP], rffi.INT,
                            save_err=rffi.RFFI_SAVE_ERRNO)
     c_getitimer = external('getitimer', [rffi.INT, itimervalP], rffi.INT)
+
+c_pthread_kill = external('pthread_kill', [lltype.Signed, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+
+if sys.platform != 'win32':
+    c_sigset_t = rffi.COpaquePtr('sigset_t', compilation_info=eci)
+    c_sigemptyset = external('sigemptyset', [c_sigset_t], rffi.INT)
+    c_sigaddset = external('sigaddset', [c_sigset_t, rffi.INT], rffi.INT)
+    c_sigismember = external('sigismember', [c_sigset_t, rffi.INT], rffi.INT)
+    c_sigwait = external('sigwait', [c_sigset_t, rffi.INTP], rffi.INT,
+                         releasegil=True,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    c_sigpending = external('sigpending', [c_sigset_t], rffi.INT,
+                            save_err=rffi.RFFI_SAVE_ERRNO)
+    c_pthread_sigmask = external('pthread_sigmask',
+                                 [rffi.INT, c_sigset_t, c_sigset_t], rffi.INT,
+                                 save_err=rffi.RFFI_SAVE_ERRNO)
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -8,10 +8,11 @@
 # XXX this does not support yet the least common AF_xxx address families
 # supported by CPython.  See http://bugs.pypy.org/issue1942
 
+from errno import EINVAL
 from rpython.rlib import _rsocket_rffi as _c, jit, rgc
 from rpython.rlib.objectmodel import instantiate, keepalive_until_here
 from rpython.rlib.rarithmetic import intmask, r_uint
-from rpython.rlib import rthread
+from rpython.rlib import rthread, rposix
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rffi import sizeof, offsetof
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -522,12 +523,28 @@
     timeout = -1.0
 
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0,
-                 fd=_c.INVALID_SOCKET):
+                 fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            fd = _c.socket(family, type, proto)
-        if _c.invalid_socket(fd):
-            raise self.error_handler()
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socket() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                fd = _c.socket(family, type | SOCK_CLOEXEC, proto)
+                if fd < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise self.error_handler()
+            if _c.invalid_socket(fd):
+                fd = _c.socket(family, type, proto)
+                if _c.invalid_socket(fd):
+                    raise self.error_handler()
+                if not inheritable:
+                    sock_set_inheritable(fd, False)
         # PLAT RISCOS
         self.fd = fd
         self.family = family
@@ -630,20 +647,33 @@
         return addr, addr.addr_p, addrlen_p
 
     @jit.dont_look_inside
-    def accept(self):
+    def accept(self, inheritable=True):
         """Wait for an incoming connection.
         Return (new socket fd, client address)."""
         if self._select(False) == 1:
             raise SocketTimeout
         address, addr_p, addrlen_p = self._addrbuf()
         try:
-            newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+            remove_inheritable = not inheritable
+            if (not inheritable and SOCK_CLOEXEC is not None
+                    and _c.HAVE_ACCEPT4
+                    and _accept4_syscall.attempt_syscall()):
+                newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
+                                         SOCK_CLOEXEC)
+                if _accept4_syscall.fallback(newfd):
+                    newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+                else:
+                    remove_inheritable = False
+            else:
+                newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
             addrlen = addrlen_p[0]
         finally:
             lltype.free(addrlen_p, flavor='raw')
             address.unlock()
         if _c.invalid_socket(newfd):
             raise self.error_handler()
+        if remove_inheritable:
+            sock_set_inheritable(newfd, False)
         address.addrlen = rffi.cast(lltype.Signed, addrlen)
         return (newfd, address)
 
@@ -1032,6 +1062,12 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
+def sock_set_inheritable(fd, inheritable):
+    try:
+        rposix.set_inheritable(fd, inheritable)
+    except OSError as e:
+        raise CSocketError(e.errno)
+
 class SocketError(Exception):
     applevelerrcls = 'error'
     def __init__(self):
@@ -1090,7 +1126,7 @@
 
 if hasattr(_c, 'socketpair'):
     def socketpair(family=socketpair_default_family, type=SOCK_STREAM, proto=0,
-                   SocketClass=RSocket):
+                   SocketClass=RSocket, inheritable=True):
         """socketpair([family[, type[, proto]]]) -> (socket object, socket object)
 
         Create a pair of socket objects from the sockets returned by the platform
@@ -1100,18 +1136,41 @@
         """
         result = lltype.malloc(_c.socketpair_t, 2, flavor='raw')
         try:
-            res = _c.socketpair(family, type, proto, result)
+            res = -1
+            remove_inheritable = not inheritable
+            if not inheritable and SOCK_CLOEXEC is not None:
+                # Non-inheritable: we try to call socketpair() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                res = _c.socketpair(family, type | SOCK_CLOEXEC,
+                                    proto, result)
+                if res < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise last_error()
+                else:
+                    remove_inheritable = False
+            #
             if res < 0:
-                raise last_error()
+                res = _c.socketpair(family, type, proto, result)
+                if res < 0:
+                    raise last_error()
             fd0 = rffi.cast(lltype.Signed, result[0])
             fd1 = rffi.cast(lltype.Signed, result[1])
         finally:
             lltype.free(result, flavor='raw')
+        if remove_inheritable:
+            sock_set_inheritable(fd0, False)
+            sock_set_inheritable(fd1, False)
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
 if _c.WIN32:
-    def dup(fd):
+    def dup(fd, inheritable=True):
         with lltype.scoped_alloc(_c.WSAPROTOCOL_INFO, zero=True) as info:
             if _c.WSADuplicateSocket(fd, rwin32.GetCurrentProcessId(), info):
                 raise last_error()
@@ -1122,15 +1181,16 @@
                 raise last_error()
             return result
 else:
-    def dup(fd):
-        return _c.dup(fd)
-
-    def fromfd(fd, family, type, proto=0, SocketClass=RSocket):
-        # Dup the fd so it and the socket can be closed independently
-        fd = _c.dup(fd)
+    def dup(fd, inheritable=True):
+        fd = rposix._dup(fd, inheritable)
         if fd < 0:
             raise last_error()
-        return make_socket(fd, family, type, proto, SocketClass)
+        return fd
+
+def fromfd(fd, family, type, proto=0, SocketClass=RSocket, inheritable=True):
+    # Dup the fd so it and the socket can be closed independently
+    fd = dup(fd, inheritable=inheritable)
+    return make_socket(fd, family, type, proto, SocketClass)
 
 def getdefaulttimeout():
     return defaults.timeout
@@ -1407,3 +1467,5 @@
     if timeout < 0.0:
         timeout = -1.0
     defaults.timeout = timeout
+
+_accept4_syscall = rposix.ENoSysCache()
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -23,16 +23,15 @@
 
 #### Constants
 
-# Identifying as _sre from Python 2.3 and onwards (at least up to 2.7)
-MAGIC = 20031017
-
 if sys.maxint > 2**32:
     MAXREPEAT = int(2**32 - 1)
+    MAXGROUPS = int(2**31 - 1)
 else:
     MAXREPEAT = int(2**31 - 1)
+    MAXGROUPS = int((2**31 / sys.maxint / 2) - 1)
 
 # In _sre.c this is bytesize of the code word type of the C implementation.
-# There it's 2 for normal Python builds and more for wide unicode builds (large 
+# There it's 2 for normal Python builds and more for wide unicode builds (large
 # enough to hold a 32-bit UCS-4 encoded character). Since here in pure Python
 # we only see re bytecodes as Python longs, we shouldn't have to care about the
 # codesize. But sre_compile will compile some stuff differently depending on the
diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -66,11 +66,13 @@
                'i': 'signed int',
                'l': 'signed long',
                'q': 'signed long long',
+               'n': 'ssize_t',
                'B': 'unsigned char',
                'H': 'unsigned short',
                'I': 'unsigned int',
                'L': 'unsigned long',
                'Q': 'unsigned long long',
+               'N': 'size_t',
                'P': 'char *',
                'f': 'float',
                'd': 'double',
@@ -78,8 +80,11 @@
                }
 
     pre_include_bits = ["""
+        #include <sys/types.h>
         #ifdef _MSC_VER
         #define _Bool char
+        typedef int ssize_t; /* XXX fixme for 64 bit*/
+        typedef unsigned int size_t; /* XXX fixme for 64 bit*/
         #endif"""]
     field_names = dict.fromkeys(INSPECT)
     for fmtchar, ctype in INSPECT.iteritems():
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -22,6 +22,16 @@
     include_dirs = [translator_c_dir],
 )
 
+class CConfig:
+    _compilation_info_ = eci
+    RPYTHREAD_NAME = rffi_platform.DefinedConstantString('RPYTHREAD_NAME')
+    USE_SEMAPHORES = rffi_platform.Defined('USE_SEMAPHORES')
+    CS_GNU_LIBPTHREAD_VERSION = rffi_platform.DefinedConstantInteger(
+        '_CS_GNU_LIBPTHREAD_VERSION')
+cconfig = rffi_platform.configure(CConfig)
+globals().update(cconfig)
+
+
 def llexternal(name, args, result, **kwds):
     kwds.setdefault('sandboxsafe', True)
     return rffi.llexternal(name, args, result, compilation_info=eci,
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1,7 +1,7 @@
 import sys
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
-from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import jit
@@ -74,7 +74,7 @@
 else:
     def code_to_unichr(code):
         # generate surrogates for large codes
-        return unichr_returns_surrogate(code)
+        return unichr_returns_surrogate(widen(code))
 
 def _STORECHAR(result, CH, byteorder):
     hi = chr(((CH) >> 8) & 0xff)
@@ -1381,7 +1381,7 @@
                 result.append(STR('\\\\'))
 
             # Map non-printable or non-ascii to '\xhh' or '\uhhhh'
-            elif pass_printable and not unicodedb.isprintable(oc):
+            elif pass_printable and not (oc <= 0x10ffff and unicodedb.isprintable(oc)):
                 char_escape_helper(result, oc)
             elif not pass_printable and (oc < 32 or oc >= 0x7F):
                 char_escape_helper(result, oc)
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -46,6 +46,7 @@
         LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP)
         LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP)
         LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.UINTP)
+        LPBOOL = rffi_platform.SimpleType("LPBOOL", rffi.LONGP)
         SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T)
         ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG)
 
@@ -58,6 +59,24 @@
         SYSTEMTIME = rffi_platform.Struct('SYSTEMTIME',
                                           [])
 
+        Struct = rffi_platform.Struct
+        COORD = Struct("COORD",
+                       [("X", rffi.SHORT),
+                        ("Y", rffi.SHORT)])
+
+        SMALL_RECT = Struct("SMALL_RECT",
+                            [("Left", rffi.SHORT),
+                             ("Top", rffi.SHORT),
+                             ("Right", rffi.SHORT),
+                             ("Bottom", rffi.SHORT)])
+
+        CONSOLE_SCREEN_BUFFER_INFO = Struct("CONSOLE_SCREEN_BUFFER_INFO",
+                                            [("dwSize", COORD),
+                                             ("dwCursorPosition", COORD),
+                                             ("wAttributes", WORD.ctype_hint),
+                                             ("srWindow", SMALL_RECT),
+                                             ("dwMaximumWindowSize", COORD)])
+
         OSVERSIONINFOEX = rffi_platform.Struct(
             'OSVERSIONINFOEX',
             [('dwOSVersionInfoSize', rffi.UINT),
@@ -92,7 +111,8 @@
                        PROCESS_VM_WRITE
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
-                       WC_NO_BEST_FIT_CHARS
+                       WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
+                       STD_ERROR_HANDLE
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -443,3 +463,13 @@
         return rffi.cast(lltype.Signed, _GetConsoleOutputCP())
 
     _wenviron_items, _wgetenv, _wputenv = make_env_impls(win32=True)
+
+
+    _GetStdHandle = winexternal(
+        'GetStdHandle', [DWORD], HANDLE)
+
+    def GetStdHandle(handle_id):
+        return _GetStdHandle(handle_id)
+    CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
+    GetConsoleScreenBufferInfo = winexternal(
+        "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -324,7 +324,8 @@
     return data
 
 
-def decompress(stream, data, flush=Z_SYNC_FLUSH, max_length=sys.maxint):
+def decompress(stream, data, flush=Z_SYNC_FLUSH, max_length=sys.maxint,
+               zdict=None):
     """
     Feed more data into an inflate stream.  Returns a tuple (string,
     finished, unused_data_length).  The string contains (a part of) the
@@ -350,7 +351,7 @@
         should_finish = False
     while_doing = "while decompressing data"
     data, err, avail_in = _operate(stream, data, flush, max_length, _inflate,
-                                   while_doing)
+                                   while_doing, zdict=zdict)
     if should_finish:
         # detect incomplete input
         rffi.setintfield(stream, 'c_avail_in', 0)
@@ -361,7 +362,7 @@
     return data, finished, avail_in
 
 
-def _operate(stream, data, flush, max_length, cfunc, while_doing):
+def _operate(stream, data, flush, max_length, cfunc, while_doing, zdict=None):
     """Common code for compress() and decompress().
     """
     # Prepare the input buffer for the stream
@@ -388,6 +389,10 @@
                 max_length -= bufsize
                 rffi.setintfield(stream, 'c_avail_out', bufsize)
                 err = cfunc(stream, flush)
+                if err == Z_NEED_DICT and zdict is not None:
+                    inflateSetDictionary(stream, zdict)
+                    # repeat the call to inflate
+                    err = cfunc(stream, flush)
                 if err == Z_OK or err == Z_STREAM_END:
                     # accumulate data into 'result'
                     avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py
--- a/rpython/rlib/test/test_rbigint.py
+++ b/rpython/rlib/test/test_rbigint.py
@@ -674,6 +674,11 @@
                 else:
                     assert ulps_check(l, math.log(op)) is None
 
+    def test_log2(self):
+        assert rbigint.fromlong(1).log(2.0) == 0.0
+        assert rbigint.fromlong(2).log(2.0) == 1.0
+        assert rbigint.fromlong(2**1023).log(2.0) == 1023.0
+
 class TestInternalFunctions(object):
     def test__inplace_divrem1(self):
         # signs are not handled in the helpers!
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -589,3 +589,18 @@
     assert rposix.get_inheritable(fd1) == False
     os.close(fd1)
     os.close(fd2)
+
+def test_SetNonInheritableCache():
+    cache = rposix.SetNonInheritableCache()
+    fd1, fd2 = os.pipe()
+    assert rposix.get_inheritable(fd1) == True
+    assert rposix.get_inheritable(fd1) == True
+    assert cache.cached_inheritable == -1
+    cache.set_non_inheritable(fd1)
+    assert cache.cached_inheritable == 1
+    cache.set_non_inheritable(fd2)
+    assert cache.cached_inheritable == 1
+    assert rposix.get_inheritable(fd1) == False
+    assert rposix.get_inheritable(fd1) == False
+    os.close(fd1)
+    os.close(fd2)
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -119,6 +119,16 @@
     s1.close()
     s2.close()
 
+def test_socketpair_inheritable():
+    if sys.platform == "win32":
+        py.test.skip('No socketpair on Windows')
+    for inh in [False, True]:
+        s1, s2 = socketpair(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        assert rposix.get_inheritable(s2.fd) == inh
+        s1.close()
+        s2.close()
+
 def test_socketpair_recvinto_1():
     class Buffer:
         def setslice(self, start, string):
@@ -378,6 +388,12 @@
     s1.close()
     s2.close()
 
+def test_inheritable():
+    for inh in [False, True]:
+        s1 = RSocket(inheritable=inh)
+        assert rposix.get_inheritable(s1.fd) == inh
+        s1.close()
+
 def test_getaddrinfo_http():
     lst = getaddrinfo('localhost', 'http')
     assert isinstance(lst, list)
diff --git a/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt b/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-3.2.0.txt
@@ -0,0 +1,912 @@
+# CaseFolding-3.2.0.txt
+# Date: 2002-03-22,20:54:33 GMT [MD]
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#65533;e" to match.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, see
+# UTR #21 Case Mappings, at http://www.unicode.org/unicode/reports/tr21/
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F2; C; 03C3; # GREEK LUNATE SIGMA SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N
+24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O
+24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P
+24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q
+24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R
+24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S
+24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T
+24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U
+24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V
+24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W
+24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X
+24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y
+24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z
+FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF
+FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI
+FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
+FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI
+FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL
+FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T
+FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST
+FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW
+FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH
+FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI
+FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW
+FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH
+FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A
+FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B
+FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C
+FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D
+FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E
+FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F
+FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G
+FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H
+FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I
+FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J
+FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K
+FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L
+FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M
+FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N
+FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O
+FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P
+FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q
+FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R
+FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S
+FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T
+FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U
+FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V
+FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W
+FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X
+FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y
+FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
+10400; C; 10428; # DESERET CAPITAL LETTER LONG I
+10401; C; 10429; # DESERET CAPITAL LETTER LONG E
+10402; C; 1042A; # DESERET CAPITAL LETTER LONG A
+10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH
+10404; C; 1042C; # DESERET CAPITAL LETTER LONG O
+10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO
+10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I
+10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E
+10408; C; 10430; # DESERET CAPITAL LETTER SHORT A
+10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH
+1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O
+1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO
+1040C; C; 10434; # DESERET CAPITAL LETTER AY
+1040D; C; 10435; # DESERET CAPITAL LETTER OW
+1040E; C; 10436; # DESERET CAPITAL LETTER WU
+1040F; C; 10437; # DESERET CAPITAL LETTER YEE
+10410; C; 10438; # DESERET CAPITAL LETTER H
+10411; C; 10439; # DESERET CAPITAL LETTER PEE
+10412; C; 1043A; # DESERET CAPITAL LETTER BEE
+10413; C; 1043B; # DESERET CAPITAL LETTER TEE
+10414; C; 1043C; # DESERET CAPITAL LETTER DEE
+10415; C; 1043D; # DESERET CAPITAL LETTER CHEE
+10416; C; 1043E; # DESERET CAPITAL LETTER JEE
+10417; C; 1043F; # DESERET CAPITAL LETTER KAY
+10418; C; 10440; # DESERET CAPITAL LETTER GAY
+10419; C; 10441; # DESERET CAPITAL LETTER EF
+1041A; C; 10442; # DESERET CAPITAL LETTER VEE
+1041B; C; 10443; # DESERET CAPITAL LETTER ETH
+1041C; C; 10444; # DESERET CAPITAL LETTER THEE
+1041D; C; 10445; # DESERET CAPITAL LETTER ES
+1041E; C; 10446; # DESERET CAPITAL LETTER ZEE
+1041F; C; 10447; # DESERET CAPITAL LETTER ESH
+10420; C; 10448; # DESERET CAPITAL LETTER ZHEE
+10421; C; 10449; # DESERET CAPITAL LETTER ER
+10422; C; 1044A; # DESERET CAPITAL LETTER EL
+10423; C; 1044B; # DESERET CAPITAL LETTER EM
+10424; C; 1044C; # DESERET CAPITAL LETTER EN
+10425; C; 1044D; # DESERET CAPITAL LETTER ENG
diff --git a/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt b/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/unicodedata/CaseFolding-5.2.0.txt
@@ -0,0 +1,1202 @@
+# CaseFolding-5.2.0.txt
+# Date: 2009-05-28, 23:02:34 GMT [MD]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2009 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For documentation, see http://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Ma&#223;e" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding 
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard, Version 5.0.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+# @missing 0000..10FFFF; <codepoint>
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX

From pypy.commits at gmail.com  Sun Aug 28 18:01:59 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 28 Aug 2016 15:01:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c35f57.28eac20a.93a56.3d44@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86665:feaeb98a9faa
Date: 2016-08-28 22:57 +0100
http://bitbucket.org/pypy/pypy/changeset/feaeb98a9faa/

Log:	hg merge default

diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20031017)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -64,7 +64,7 @@
         AsyncAction.__init__(self, space)
         self.pending_signal = -1
         self.fire_in_another_thread = False
-        #
+
         @rgc.no_collect
         def _after_thread_switch():
             if self.fire_in_another_thread:
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -23,10 +23,6 @@
 
 #### Constants
 
-# Identifying as _sre from Python 2.3 and onwards (at least up to 2.7)
-# UPDATE: change was necessary for Python 3.3 changes
-MAGIC = 20140917
-
 if sys.maxint > 2**32:
     MAXREPEAT = int(2**32 - 1)
     MAXGROUPS = int(2**31 - 1)
@@ -35,7 +31,7 @@
     MAXGROUPS = int((2**31 / sys.maxint / 2) - 1)
 
 # In _sre.c this is bytesize of the code word type of the C implementation.
-# There it's 2 for normal Python builds and more for wide unicode builds (large 
+# There it's 2 for normal Python builds and more for wide unicode builds (large
 # enough to hold a 32-bit UCS-4 encoded character). Since here in pure Python
 # we only see re bytecodes as Python longs, we shouldn't have to care about the
 # codesize. But sre_compile will compile some stuff differently depending on the

From pypy.commits at gmail.com  Sun Aug 28 18:02:01 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 28 Aug 2016 15:02:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c35f59.a6a5c20a.b957f.365a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r86666:c34251a385be
Date: 2016-08-28 22:58 +0100
http://bitbucket.org/pypy/pypy/changeset/c34251a385be/

Log:	hg merge py3k

diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20140917)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'MAXGROUPS':      'space.wrap(interp_sre.MAXGROUPS)',
         'compile':        'interp_sre.W_SRE_Pattern',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -64,7 +64,7 @@
         AsyncAction.__init__(self, space)
         self.pending_signal = -1
         self.fire_in_another_thread = False
-        #
+
         @rgc.no_collect
         def _after_thread_switch():
             if self.fire_in_another_thread:
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -23,10 +23,6 @@
 
 #### Constants
 
-# Identifying as _sre from Python 2.3 and onwards (at least up to 2.7)
-# UPDATE: change was necessary for Python 3.3 changes
-MAGIC = 20140917
-
 if sys.maxint > 2**32:
     MAXREPEAT = int(2**32 - 1)
     MAXGROUPS = int(2**31 - 1)
@@ -35,7 +31,7 @@
     MAXGROUPS = int((2**31 / sys.maxint / 2) - 1)
 
 # In _sre.c this is bytesize of the code word type of the C implementation.
-# There it's 2 for normal Python builds and more for wide unicode builds (large 
+# There it's 2 for normal Python builds and more for wide unicode builds (large
 # enough to hold a 32-bit UCS-4 encoded character). Since here in pure Python
 # we only see re bytecodes as Python longs, we shouldn't have to care about the
 # codesize. But sre_compile will compile some stuff differently depending on the

From pypy.commits at gmail.com  Sun Aug 28 18:04:18 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 28 Aug 2016 15:04:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backport rpython changes from py3k
Message-ID: <57c35fe2.2916c20a.a4535.3c14@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86667:608532451f21
Date: 2016-08-28 23:03 +0100
http://bitbucket.org/pypy/pypy/changeset/608532451f21/

Log:	Backport rpython changes from py3k

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1172,14 +1172,11 @@
         if ok:
             fdread = c_open_osfhandle(hread, 0)
             fdwrite = c_open_osfhandle(hwrite, 1)
-            if fdread == -1 or fdwrite == -1:
-                rwin32.CloseHandle(hread)
-                rwin32.CloseHandle(hwrite)
-                ok = 0
-        if not ok:
-            raise WindowsError(rwin32.GetLastError_saved(),
-                               "CreatePipe failed")
-        return (fdread, fdwrite)
+            if not (fdread == -1 or fdwrite == -1):
+                return (fdread, fdwrite)
+            rwin32.CloseHandle(pread)
+            rwin32.CloseHandle(pwrite)
+        raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
@@ -2142,6 +2139,7 @@
 eci_inheritable = eci.merge(ExternalCompilationInfo(
     separate_module_sources=[r"""
 #include <errno.h>
+#include <sys/ioctl.h>
 
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)

From pypy.commits at gmail.com  Sun Aug 28 19:48:38 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 28 Aug 2016 16:48:38 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: PyMemoryView_GET_BUFFER wip,
 len(memorview()) is prod(shape), but getlength() is number of bytes
Message-ID: <57c37856.4317c20a.8297b.5e3e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86668:3ec8f09e764d
Date: 2016-08-29 09:24 +1000
http://bitbucket.org/pypy/pypy/changeset/3ec8f09e764d/

Log:	PyMemoryView_GET_BUFFER wip, len(memorview()) is prod(shape), but
	getlength() is number of bytes

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,7 +1,8 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                                    build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+                                    build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, as_pyobj, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.objspace.std.memoryobject import W_MemoryView
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
@@ -20,21 +21,29 @@
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
     view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-    # TODO - fill in fields
-    '''
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if not isinstance(w_obj, W_MemoryView):
+        return view
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+    except ValueError:
+        return view
+    view.c_len = w_obj.getlength()
+    view.c_obj = as_pyobj(space, w_obj)
+    incref(space, view.c_obj)
+    view.c_itemsize = w_obj.buf.getitemsize()
+    rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+    ndim = w_obj.buf.getndim()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.str2charp(w_obj.buf.getformat())
+    view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    shape = w_obj.buf.getshape()
+    strides = w_obj.buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    ''' 
     return view
 
 
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -508,10 +508,9 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Releases a Py_buffer obtained from getbuffer ParseTuple's s*.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
+    Release the buffer view. This should be called when the buffer is 
+    no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)
     view.c_obj = lltype.nullptr(PyObject.TO)
+    # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -21,6 +21,7 @@
         y = memoryview(arr)
         assert y.format == 'i'
         assert y.shape == (10,)
+        assert len(y) == 10
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -115,7 +115,7 @@
             self.buf.setslice(start, value.as_str())
 
     def descr_len(self, space):
-        return space.wrap(self.buf.getlength())
+        return space.wrap(self.buf.getlength() / self.buf.getitemsize())
 
     def w_get_format(self, space):
         return space.wrap(self.buf.getformat())

From pypy.commits at gmail.com  Sun Aug 28 20:00:13 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 28 Aug 2016 17:00:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix: save the handles for CloseHandle
Message-ID: <57c37b0d.11331c0a.14c66.8917@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r86669:16f19728001a
Date: 2016-08-28 16:59 -0700
http://bitbucket.org/pypy/pypy/changeset/16f19728001a/

Log:	fix: save the handles for CloseHandle

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1159,23 +1159,18 @@
     # 'flags' might be ignored.  Check the result.
     if _WIN32:
         # 'flags' ignored
-        pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        try:
-            ok = CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0)
-            hread = rffi.cast(rffi.INTPTR_T, pread[0])
-            hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
-        finally:
-            lltype.free(pwrite, flavor='raw')
-            lltype.free(pread, flavor='raw')
-        if ok:
-            fdread = c_open_osfhandle(hread, 0)
-            fdwrite = c_open_osfhandle(hwrite, 1)
-            if not (fdread == -1 or fdwrite == -1):
-                return (fdread, fdwrite)
-            rwin32.CloseHandle(pread)
-            rwin32.CloseHandle(pwrite)
+        ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        with ralloc as pread, walloc as pwrite:
+            if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
+                fdread = c_open_osfhandle(
+                    rffi.cast(rffi.INTPTR_T, pread[0]), 0)
+                fdwrite = c_open_osfhandle(
+                    rffi.cast(rffi.INTPTR_T, pwrite[0]), 1)
+                if not (fdread == -1 or fdwrite == -1):
+                    return (fdread, fdwrite)
+                rwin32.CloseHandle(pread)
+                rwin32.CloseHandle(pwrite)
         raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')

From pypy.commits at gmail.com  Sun Aug 28 20:01:12 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 28 Aug 2016 17:01:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix: save the handles for CloseHandle
Message-ID: <57c37b48.4abf1c0a.79d0a.8ee8@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r86670:ee05b358fdec
Date: 2016-08-28 16:59 -0700
http://bitbucket.org/pypy/pypy/changeset/ee05b358fdec/

Log:	fix: save the handles for CloseHandle (grafted from
	16f19728001ade4a9f35b3d15521db4fb85d8a27)

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1159,23 +1159,18 @@
     # 'flags' might be ignored.  Check the result.
     if _WIN32:
         # 'flags' ignored
-        pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        try:
-            ok = CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0)
-            hread = rffi.cast(rffi.INTPTR_T, pread[0])
-            hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
-        finally:
-            lltype.free(pwrite, flavor='raw')
-            lltype.free(pread, flavor='raw')
-        if ok:
-            fdread = c_open_osfhandle(hread, 0)
-            fdwrite = c_open_osfhandle(hwrite, 1)
-            if not (fdread == -1 or fdwrite == -1):
-                return (fdread, fdwrite)
-            rwin32.CloseHandle(pread)
-            rwin32.CloseHandle(pwrite)
+        ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        with ralloc as pread, walloc as pwrite:
+            if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
+                fdread = c_open_osfhandle(
+                    rffi.cast(rffi.INTPTR_T, pread[0]), 0)
+                fdwrite = c_open_osfhandle(
+                    rffi.cast(rffi.INTPTR_T, pwrite[0]), 1)
+                if not (fdread == -1 or fdwrite == -1):
+                    return (fdread, fdwrite)
+                rwin32.CloseHandle(pread)
+                rwin32.CloseHandle(pwrite)
         raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')

From pypy.commits at gmail.com  Sun Aug 28 23:11:10 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 28 Aug 2016 20:11:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix
Message-ID: <57c3a7ce.4abf1c0a.79d0a.b2c2@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r86671:0d491154d501
Date: 2016-08-28 20:09 -0700
http://bitbucket.org/pypy/pypy/changeset/0d491154d501/

Log:	fix

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1159,8 +1159,8 @@
     # 'flags' might be ignored.  Check the result.
     if _WIN32:
         # 'flags' ignored
-        ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
+        walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
         with ralloc as pread, walloc as pwrite:
             if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
                 fdread = c_open_osfhandle(

From pypy.commits at gmail.com  Sun Aug 28 23:11:12 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 28 Aug 2016 20:11:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <57c3a7d0.274fc20a.2eca5.84b8@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r86672:8d7e62c4cbd9
Date: 2016-08-28 20:10 -0700
http://bitbucket.org/pypy/pypy/changeset/8d7e62c4cbd9/

Log:	merge default

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1159,8 +1159,8 @@
     # 'flags' might be ignored.  Check the result.
     if _WIN32:
         # 'flags' ignored
-        ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
+        ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
+        walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
         with ralloc as pread, walloc as pwrite:
             if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
                 fdread = c_open_osfhandle(

From pypy.commits at gmail.com  Mon Aug 29 03:54:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 00:54:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Hack at importlib/_bootstrap_external.py
 to use the same magic version
Message-ID: <57c3ea28.11051c0a.563fc.011d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86673:cea8ccfe05f4
Date: 2016-08-29 09:53 +0200
http://bitbucket.org/pypy/pypy/changeset/cea8ccfe05f4/

Log:	Hack at importlib/_bootstrap_external.py to use the same magic
	version number for pyc files as the one declared in
	pypy/interpreter/pycode.py.

diff --git a/lib-python/3/importlib/_bootstrap_external.py b/lib-python/3/importlib/_bootstrap_external.py
--- a/lib-python/3/importlib/_bootstrap_external.py
+++ b/lib-python/3/importlib/_bootstrap_external.py
@@ -228,7 +228,14 @@
 # longer be understood by older implementations of the eval loop (usually
 # due to the addition of new opcodes).
 
-MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n'
+# MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n'
+#
+# PyPy change: the MAGIC_NUMBER is defined in
+# pypy/interpreter/pycode.py, 'default_magic'.  It is based on a number
+# different than CPython's, always < 3000.  We get the 4-bytes string
+# here via a hack: MAGIC_NUMBER is set in the module from
+# module/_frozen_importlib/__init__.py before the module is executed.
+
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -35,7 +35,8 @@
 
 # we compute the magic number in a similar way to CPython, but we use a
 # different value for the highest 16 bits. Bump pypy_incremental_magic every
-# time you make pyc files incompatible
+# time you make pyc files incompatible.  This value ends up in the frozen
+# importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
 pypy_incremental_magic = 80 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -28,10 +28,15 @@
 
     def install(self):
         """NOT_RPYTHON"""
+        from pypy.module.imp import interp_imp
+
         super(Module, self).install()
         space = self.space
         # "import importlib/_boostrap_external.py"
         w_mod = Module(space, space.wrap("_frozen_importlib_external"))
+        # hack: inject MAGIC_NUMBER into this module's dict
+        space.setattr(w_mod, space.wrap('MAGIC_NUMBER'),
+                      interp_imp.get_magic(space))
         self._compile_bootstrap_module(
             space, '_bootstrap_external', w_mod.w_name, w_mod.w_dict)
         space.sys.setmodule(w_mod)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -24,7 +24,11 @@
 PREFIX = 'pypy3-'
 DEFAULT_SOABI = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])
 
-PYC_TAG = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])
+PYC_TAG = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])   # 'pypy3-XY'
+
+# see also pypy_incremental_magic in interpreter/pycode.py for the magic
+# version number stored inside pyc files.
+
 
 @specialize.memo()
 def get_so_extension(space):

From pypy.commits at gmail.com  Mon Aug 29 04:43:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 01:43:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Test and fix (and remove the temporary
 dirty fix which only works if
Message-ID: <57c3f59a.a719c20a.7a270.dd0f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86674:35fc02e01bf7
Date: 2016-08-29 10:42 +0200
http://bitbucket.org/pypy/pypy/changeset/35fc02e01bf7/

Log:	Test and fix (and remove the temporary dirty fix which only works if
	there is a list object around)

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1198,7 +1198,7 @@
                 self.settopvalue(self.space.w_None)
 
     @jit.unroll_safe
-    def call_function(self, oparg, w_star=None, w_starstar=None):
+    def call_function(self, oparg, w_starstar=None, has_vararg=False):
         n_arguments = oparg & 0xff
         n_keywords = (oparg>>8) & 0xff
         if n_keywords:
@@ -1210,20 +1210,16 @@
                     break
                 w_value = self.popvalue()
                 w_key = self.popvalue()
-                # temporary (dirty) fix: if star-arg occurs after kwarg,
-                # arg order is reversed on stack
-                from pypy.objspace.std.listobject import W_ListObject
-                if isinstance(w_key, W_ListObject):
-                    w_key_temp = w_key
-                    w_key = w_value
-                    w_value = w_star
-                    w_star = w_key_temp
                 key = self.space.identifier_w(w_key)
                 keywords[n_keywords] = key
                 keywords_w[n_keywords] = w_value
         else:
             keywords = None
             keywords_w = None
+        if has_vararg:
+            w_star = self.popvalue()
+        else:
+            w_star = None
         arguments = self.popvalues(n_arguments)
         args = self.argument_factory(arguments, keywords, keywords_w, w_star,
                                      w_starstar)
@@ -1252,17 +1248,15 @@
             self.call_function(oparg)
 
     def CALL_FUNCTION_VAR(self, oparg, next_instr):
-        w_varargs = self.popvalue()
-        self.call_function(oparg, w_varargs)
+        self.call_function(oparg, has_vararg=True)
 
     def CALL_FUNCTION_KW(self, oparg, next_instr):
         w_varkw = self.popvalue()
-        self.call_function(oparg, None, w_varkw)
+        self.call_function(oparg, w_varkw)
 
     def CALL_FUNCTION_VAR_KW(self, oparg, next_instr):
         w_varkw = self.popvalue()
-        w_varargs = self.popvalue()
-        self.call_function(oparg, w_varargs, w_varkw)
+        self.call_function(oparg, w_varkw, has_vararg=True)
 
     @jit.unroll_safe
     def _make_function(self, oparg, freevars=None):
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -214,6 +214,15 @@
         assert self.codetest(code, 'g', [12, {}]) ==    ()
         assert self.codetest(code, 'g', [12, {3:1}]) == (3,)
 
+    def test_star_arg_after_keyword_arg(self):
+        code = '''
+            def f(a, b):
+                return a - b
+            def g(a, b):
+                return f(b=b, *(a,))
+        '''
+        assert self.codetest(code, 'g', [40, 2]) == 38
+
     def test_closure(self):
         code = '''
             def f(x, y):

From pypy.commits at gmail.com  Mon Aug 29 05:03:19 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 02:03:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: progress on implementing
 descr_cast 1 -> N dimensions, translation fixes
Message-ID: <57c3fa57.a427c20a.17051.dfe4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86675:97a4f9949ccd
Date: 2016-08-29 11:02 +0200
http://bitbucket.org/pypy/pypy/changeset/97a4f9949ccd/

Log:	progress on implementing descr_cast 1 -> N dimensions, translation
	fixes

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -26,18 +26,43 @@
     an interp-level buffer.
     """
 
-    def __init__(self, buf, format=None, itemsize=1):
+    def __init__(self, buf, format=None, itemsize=1, ndim=-1,
+                 shape=None, strides=None, suboffsets=None):
         assert isinstance(buf, Buffer)
         self.buf = buf
         self._hash = -1
         self.format = format
         self.itemsize = itemsize
+        self.shape = shape
+        self.strides = strides
+        self.suboffsets = suboffsets
+        self.ndim = ndim
         self.flags = 0
         self._init_flags()
 
+    # several fields are "overwritten" by the memory view (shape, strides, ...)
+    # thus use only those getter fields instead of directly accessing the fields
+    def getndim(self):
+        if self.ndim == -1:
+            return self.buf.getndim()
+        return self.ndim
+
+    def getshape(self):
+        if self.shape is None:
+            return self.buf.getshape()
+        return self.shape
+
+    def getstrides(self):
+        if self.strides is None:
+            return self.buf.getstrides()
+        return self.strides
+
+    def getitemsize(self):
+        return self.itemsize
+
+    # memoryview needs to modify the field 'format', to prevent the modification
+    # of the buffer, we save the new format here!
     def getformat(self):
-        # memoryview needs to modify the field 'format', to prevent the modification
-        # of the buffer, we save the new format here!
         if self.format is None:
             return self.buf.getformat()
         return self.format
@@ -94,18 +119,51 @@
 
     def descr_tolist(self, space):
         self._check_released(space)
+
+        buf = self.buf
+        dim = buf.getndim()
+        fmt = self.getformat()
+        if dim == 0:
+            raise NotImplementedError
+        elif dim == 1:
+            return self._tolist(space, buf, self.getlength(), fmt)
+        else:
+            return self._tolist_rec(space, buf, fmt)
+
+    def _tolist(self, space, buf, count, fmt):
         # TODO: this probably isn't very fast
         fmtiter = UnpackFormatIterator(space, self.buf)
-        fmtiter.interpret(self.format * self.getlength())
+        fmtiter.interpret(fmt * count)
         return space.newlist(fmtiter.result_w)
 
+    def _tolist_rec(self, space, buf, start, dim, fmt):
+        idim = dim-1
+        strides = self.getstrides()
+        stride = strides[idim]
+        itemsize = self.getitemsize()
+        if dim >= buf.getndim():
+            return self._tolist(space, SubBuffer(buf, start, itemsize), stride // itemsize, fmt)
+        shape = self.getshape()
+        dimshape = shape[idim]
+        items = [None] * dimshape
+
+        for i in range(dimshape):
+            item = self._tolist_rec(space, SubBuffer(buf, start, self.itemsize), dim+1, fmt)
+            items[i] = item
+            start += stride
+
+        return space.newlist(items,len(items))
+
+
     def _start_from_tuple(self, space, w_tuple):
+        from pypy.objspace.std.tupleobject import W_TupleObject
         start = 0
 
         view = self.buf
         length = space.len_w(w_tuple)
         dim = view.getndim()
         dim = 0
+        assert isinstance(w_tuple, W_TupleObject)
         while dim < length:
             w_obj = w_tuple.getitem(space, dim)
             index = w_obj.int_w(space)
@@ -166,7 +224,7 @@
             # XXX why? returns a memory view on int index if step == 0:
             #    step = 1
 
-            # start & stop are now byte offset, thus use self.bug.getlength()
+            # start & stop are now byte offset, thus use self.buf.getlength()
             if stop > self.buf.getlength():
                 raise oefmt(space.w_IndexError, 'index out of range')
         if step not in (0, 1):
@@ -208,7 +266,7 @@
             # TODO: this probably isn't very fast
             fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)
             try:
-                fmtiter.interpret(self.format)
+                fmtiter.interpret(self.getformat())
             except StructError as e:
                 raise oefmt(space.w_TypeError,
                             "memoryview: invalid type for format '%s'",
@@ -328,10 +386,10 @@
         return size
 
     def _zero_in_shape(self):
-        # TODO move to buffer
-        view = self.buf
-        shape = view.shape
-        for i in range(view.ndim):
+        # this method could be moved to the class Buffer
+        buf = self.buf
+        shape = buf.getshape()
+        for i in range(buf.getndim()):
             if shape[i] == 0:
                 return True
         return False
@@ -359,14 +417,15 @@
 
         itemsize = self.get_native_fmtchar(fmt)
         if w_shape:
-            if not (space.is_w(w_obj, space.w_list) or space.is_w(w_obj, space.w_tuple)):
-                raise oefmt(space.w_TypeError, "expected list or tuple got %T", w_obj)
-            ndim = space.len_w(w_obj)
-            if ndim > space.BUF_MAX_DIM:
+            if not (space.isinstance_w(w_shape, space.w_list) or space.isinstance_w(w_shape, space.w_tuple)):
+                raise oefmt(space.w_TypeError, "expected list or tuple got %T", w_shape)
+            ndim = space.len_w(w_shape)
+            if ndim > MEMORYVIEW_MAX_DIM:
                 raise oefmt(space.w_ValueError, \
                         "memoryview: number of dimensions must not exceed %d",
                         ndim)
-            if ndim != buf.ndim:
+            # yes access ndim as field
+            if self.ndim > 1 and buf.getndim() != 1:
                 raise OperationError(space.w_TypeError, \
                     space.wrap("memoryview: cast must be 1D -> ND or ND -> 1D"))
 
@@ -374,8 +433,8 @@
         origfmt = mv.getformat()
         mv._cast_to_1D(space, origfmt, fmt, itemsize)
         if w_shape:
-            shape = [space.int_w(w_obj) for w_obj in w_shape.fixedview_unroll()]
-            mv._cast_to_ND(space, shape, dim)
+            shape = [space.int_w(w_obj) for w_obj in w_shape.getitems_unroll()]
+            mv._cast_to_ND(space, shape, ndim)
         return mv
 
     def _init_flags(self):
@@ -426,7 +485,7 @@
         self.itemsize = itemsize
         self.ndim = 1
         self.shape = [buf.getlength() // buf.getitemsize()]
-        self.srides = [buf.getitemsize()]
+        self.strides = [buf.getitemsize()]
         # XX suboffsets
 
         self._init_flags()
@@ -457,7 +516,32 @@
         return None
 
     def _cast_to_ND(self, space, shape, ndim):
-        pass
+        buf = self.buf
+
+        self.ndim = ndim
+        length = self.itemsize
+        if ndim == 0:
+            self.shape = []
+            self.strides = []
+        else:
+            self.shape = shape
+            for i in range(ndim):
+                length *= shape[i]
+            self._init_strides_from_shape()
+
+        if length != self.buf.getlength():
+            raise OperationError(space.w_TypeError,
+                    space.wrap("memoryview: product(shape) * itemsize != buffer size"))
+
+        self._init_flags()
+
+    def _init_strides_from_shape(self):
+        s = [0] * len(self.shape)
+        self.strides = s
+        dim = self.getndim()
+        s[dim-1] = self.itemsize
+        for i in range(0,ndim-2,-1):
+            s[i] = s[i+1] * shape[i+1]
 
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -199,7 +199,10 @@
                 for j, w_obj in enumerate(w_work.getitems_unroll()):
                     worklist.insert(0, (dim+1, w_obj))
                 continue
-            self.data.append(space.int_w(w_work))
+            byte = struct.pack(self.format, space.int_w(w_work))
+            for c in byte:
+                self.data.append(c)
+        self.data = ''.join(self.data)
 
     def getslice(self, start, stop, step, size):
         items = []
@@ -218,10 +221,10 @@
         return self.format
 
     def getitem(self, index):
-        return struct.pack(self.format, self.data[index])
+        return self.data[index:index+1]
 
     def getlength(self):
-        return len(self.data) * self.itemsize
+        return len(self.data)
 
     def getitemsize(self):
         return self.itemsize
@@ -329,3 +332,14 @@
         except TypeError:
             pass
 
+    def test_cast_with_shape(self):
+        empty = self.MockArray([1,0,2,0,3,0],
+                    dim=1, fmt='h', size=2,
+                    strides=[8], shape=[6])
+        view = memoryview(empty)
+        byteview = view.cast('b')
+        assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]
+        i32view = byteview.cast('i', shape=[1,3])
+        assert i32view.format == 'i'
+        assert i32view.itemsize == 4
+        assert i32view.tolist() == [[1,2,3]]

From pypy.commits at gmail.com  Mon Aug 29 05:14:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 02:14:29 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: nothing to do specifically for enum
Message-ID: <57c3fcf5.915c1c0a.f8309.244d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5688:b22a689d00ed
Date: 2016-08-29 11:14 +0200
http://bitbucket.org/pypy/extradoc/changeset/b22a689d00ed/

Log:	nothing to do specifically for enum

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -74,5 +74,7 @@
   depends on compact unicode representation)
 
 * enum: Support for enumeration types (PEP 435).
+  (PURELY-APPLEVEL, nothing to do, except look at test failures showing
+  other things not implemented like always)
 
 * pathlib: Object-oriented filesystem paths (PEP 428).

From pypy.commits at gmail.com  Mon Aug 29 05:16:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 02:16:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Be more careful about encoding/decoding
 to utf-8, as even the
Message-ID: <57c3fd5a.10a81c0a.da4cd.2553@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86676:9d26d61f920a
Date: 2016-08-29 11:15 +0200
http://bitbucket.org/pypy/pypy/changeset/9d26d61f920a/

Log:	Be more careful about encoding/decoding to utf-8, as even the
	.encode('utf-8') crashes if given the unichar that is normally used
	for surrogates

diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -371,9 +371,9 @@
     m.atom_str(TYPE_STRING, x.co_code)
     _marshal_tuple(space, x.co_consts_w, m)
     _marshal_tuple(space, x.co_names_w, m)   # list of w_unicodes
-    co_varnames_w = [space.wrap(s.decode('utf-8')) for s in x.co_varnames]
-    co_freevars_w = [space.wrap(s.decode('utf-8')) for s in x.co_freevars]
-    co_cellvars_w = [space.wrap(s.decode('utf-8')) for s in x.co_cellvars]
+    co_varnames_w = [space.wrap(_decode_utf8(space, s)) for s in x.co_varnames]
+    co_freevars_w = [space.wrap(_decode_utf8(space, s)) for s in x.co_freevars]
+    co_cellvars_w = [space.wrap(_decode_utf8(space, s)) for s in x.co_cellvars]
     _marshal_tuple(space, co_varnames_w, m)  # more lists, now of w_unicodes
     _marshal_tuple(space, co_freevars_w, m)
     _marshal_tuple(space, co_cellvars_w, m)
@@ -387,7 +387,8 @@
 
 def _unmarshal_strlist(u):
     items_w = _unmarshal_tuple_w(u)
-    return [u.space.unicode_w(w_item).encode('utf-8') for w_item in items_w]
+    return [_encode_utf8(u.space, u.space.unicode_w(w_item))
+            for w_item in items_w]
 
 def _unmarshal_tuple_w(u):
     w_obj = u.get_w_obj()
@@ -413,8 +414,8 @@
     varnames    = _unmarshal_strlist(u)
     freevars    = _unmarshal_strlist(u)
     cellvars    = _unmarshal_strlist(u)
-    filename    = space.unicode0_w(u.get_w_obj()).encode('utf-8')
-    name        = space.unicode_w(u.get_w_obj()).encode('utf-8')
+    filename    = _encode_utf8(space, space.unicode0_w(u.get_w_obj()))
+    name        = _encode_utf8(space, space.unicode_w(u.get_w_obj()))
     firstlineno = u.get_int()
     lnotab      = space.bytes_w(u.get_w_obj())
     PyCode.__init__(w_codeobj,
@@ -439,15 +440,20 @@
     if typecode != FLAG_DONE:
         m.atom_str(typecode, s)
 
+def _encode_utf8(space, u):
+    return unicodehelper.encode_utf8(space, u, allow_surrogates=True)
+
+def _decode_utf8(space, s):
+    return unicodehelper.decode_utf8(space, s, allow_surrogates=True)
+
 @marshaller(W_UnicodeObject)
 def marshal_unicode(space, w_unicode, m):
-    s = unicodehelper.encode_utf8(space, space.unicode_w(w_unicode),
-                                  allow_surrogates=True)
+    s = _encode_utf8(space, space.unicode_w(w_unicode))
     _marshal_unicode(space, s, m, w_unicode=w_unicode)
 
 @unmarshaller(TYPE_UNICODE)
 def unmarshal_unicode(space, u, tc):
-    uc = unicodehelper.decode_utf8(space, u.get_str(), allow_surrogates=True)
+    uc = _decode_utf8(space, u.get_str())
     return space.newunicode(uc)
 
 @unmarshaller(TYPE_INTERNED)

From pypy.commits at gmail.com  Mon Aug 29 05:26:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 02:26:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: translation fix
Message-ID: <57c3ffd3.434bc20a.8dc9.ac17@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86677:5acad0a7aef3
Date: 2016-08-29 11:26 +0200
http://bitbucket.org/pypy/pypy/changeset/5acad0a7aef3/

Log:	translation fix

diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -1,5 +1,5 @@
 from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.rstring import StringBuilder, assert_str0
 from rpython.rlib.rstruct import ieee
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib import objectmodel
@@ -418,6 +418,7 @@
     name        = _encode_utf8(space, space.unicode_w(u.get_w_obj()))
     firstlineno = u.get_int()
     lnotab      = space.bytes_w(u.get_w_obj())
+    filename = assert_str0(filename)
     PyCode.__init__(w_codeobj,
                   space, argcount, kwonlyargcount, nlocals, stacksize, flags,
                   code, consts_w[:], names, varnames, filename,

From pypy.commits at gmail.com  Mon Aug 29 05:50:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 02:50:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix for test_kw_defaults_None
Message-ID: <57c40581.64f9c20a.aba20.b3b2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86678:2039ed00c6cd
Date: 2016-08-29 11:50 +0200
http://bitbucket.org/pypy/pypy/changeset/2039ed00c6cd/

Log:	Fix for test_kw_defaults_None

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -432,7 +432,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(w_decorator_list)
         _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        _returns = expr.from_object(space, w_returns) if w_returns is not None else None
+        _returns = expr.from_object(space, w_returns)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return FunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset)
@@ -508,7 +508,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(w_decorator_list)
         _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        _returns = expr.from_object(space, w_returns) if w_returns is not None else None
+        _returns = expr.from_object(space, w_returns)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return AsyncFunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset)
@@ -630,7 +630,7 @@
         w_value = get_field(space, w_node, 'value', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _value = expr.from_object(space, w_value) if w_value is not None else None
+        _value = expr.from_object(space, w_value)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Return(_value, _lineno, _col_offset)
@@ -1190,8 +1190,8 @@
         w_cause = get_field(space, w_node, 'cause', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _exc = expr.from_object(space, w_exc) if w_exc is not None else None
-        _cause = expr.from_object(space, w_cause) if w_cause is not None else None
+        _exc = expr.from_object(space, w_exc)
+        _cause = expr.from_object(space, w_cause)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Raise(_exc, _cause, _lineno, _col_offset)
@@ -1314,7 +1314,7 @@
         _test = expr.from_object(space, w_test)
         if _test is None:
             raise_required_value(space, w_node, 'test')
-        _msg = expr.from_object(space, w_msg) if w_msg is not None else None
+        _msg = expr.from_object(space, w_msg)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Assert(_test, _msg, _lineno, _col_offset)
@@ -2312,7 +2312,7 @@
         w_value = get_field(space, w_node, 'value', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _value = expr.from_object(space, w_value) if w_value is not None else None
+        _value = expr.from_object(space, w_value)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Yield(_value, _lineno, _col_offset)
@@ -3101,9 +3101,9 @@
         w_lower = get_field(space, w_node, 'lower', True)
         w_upper = get_field(space, w_node, 'upper', True)
         w_step = get_field(space, w_node, 'step', True)
-        _lower = expr.from_object(space, w_lower) if w_lower is not None else None
-        _upper = expr.from_object(space, w_upper) if w_upper is not None else None
-        _step = expr.from_object(space, w_step) if w_step is not None else None
+        _lower = expr.from_object(space, w_lower)
+        _upper = expr.from_object(space, w_upper)
+        _step = expr.from_object(space, w_step)
         return Slice(_lower, _upper, _step)
 
 State.ast_type('Slice', 'slice', ['lower', 'upper', 'step'])
@@ -3583,7 +3583,7 @@
         w_body = get_field(space, w_node, 'body', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _type = expr.from_object(space, w_type) if w_type is not None else None
+        _type = expr.from_object(space, w_type)
         _name = space.str_or_None_w(w_name)
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
@@ -3664,12 +3664,12 @@
         w_defaults = get_field(space, w_node, 'defaults', False)
         args_w = space.unpackiterable(w_args)
         _args = [arg.from_object(space, w_item) for w_item in args_w]
-        _vararg = arg.from_object(space, w_vararg) if w_vararg is not None else None
+        _vararg = arg.from_object(space, w_vararg) if not space.is_w(w_vararg, space.w_None) else None
         kwonlyargs_w = space.unpackiterable(w_kwonlyargs)
         _kwonlyargs = [arg.from_object(space, w_item) for w_item in kwonlyargs_w]
         kw_defaults_w = space.unpackiterable(w_kw_defaults)
         _kw_defaults = [expr.from_object(space, w_item) for w_item in kw_defaults_w]
-        _kwarg = arg.from_object(space, w_kwarg) if w_kwarg is not None else None
+        _kwarg = arg.from_object(space, w_kwarg) if not space.is_w(w_kwarg, space.w_None) else None
         defaults_w = space.unpackiterable(w_defaults)
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwonlyargs, _kw_defaults, _kwarg, _defaults)
@@ -3705,7 +3705,7 @@
         _arg = space.identifier_w(w_arg)
         if _arg is None:
             raise_required_value(space, w_node, 'arg')
-        _annotation = expr.from_object(space, w_annotation) if w_annotation is not None else None
+        _annotation = expr.from_object(space, w_annotation)
         return arg(_arg, _annotation)
 
 State.ast_type('arg', 'AST', ['arg', 'annotation'])
@@ -3805,7 +3805,7 @@
         _context_expr = expr.from_object(space, w_context_expr)
         if _context_expr is None:
             raise_required_value(space, w_node, 'context_expr')
-        _optional_vars = expr.from_object(space, w_optional_vars) if w_optional_vars is not None else None
+        _optional_vars = expr.from_object(space, w_optional_vars)
         return withitem(_context_expr, _optional_vars)
 
 State.ast_type('withitem', 'AST', ['context_expr', 'optional_vars'])
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -160,7 +160,17 @@
         else:
             extractor = "%s.from_object(space, %s)" % (field.type, value)
             if field.opt:
-                extractor += " if %s is not None else None" % (value,)
+                if field.type == 'expr':
+                    # the expr.from_object() method should accept w_None and
+                    # return None; nothing more to do here
+                    pass
+                elif field.type == 'arg':
+                    # the method arg.from_object() doesn't accept w_None
+                    extractor += (
+                        ' if not space.is_w(%s, space.w_None) else None'
+                        % (value,))
+                else:
+                    raise NotImplementedError(field.type)
             return extractor
 
     def get_field_converter(self, field):

From pypy.commits at gmail.com  Mon Aug 29 05:55:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 02:55:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix test
Message-ID: <57c406ae.28eac20a.93a56.fadb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86679:549d60af4953
Date: 2016-08-29 11:55 +0200
http://bitbucket.org/pypy/pypy/changeset/549d60af4953/

Log:	Fix test

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -864,9 +864,10 @@
         with a: pass
         with a: pass
         with a: pass
+        with a: pass
         """
         code = compile_with_astcompiler(source, 'exec', self.space)
-        assert code.co_stacksize == 5
+        assert code.co_stacksize == 6  # i.e. <= 7, there is no systematic leak
 
     def test_stackeffect_bug5(self):
         source = """if 1:

From pypy.commits at gmail.com  Mon Aug 29 06:05:32 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 03:05:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: descr_tolist now returns a
 nested list constructed using shape + dim of the memoryview,
 memoryview now also carries shape,
 strides and dim as fields to make this possible
Message-ID: <57c408ec.8628c20a.5d8b9.016c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86680:4dfc3427fc69
Date: 2016-08-29 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/4dfc3427fc69/

Log:	descr_tolist now returns a nested list constructed using shape + dim
	of the memoryview, memoryview now also carries shape,strides and dim
	as fields to make this possible

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -121,38 +121,44 @@
         self._check_released(space)
 
         buf = self.buf
-        dim = buf.getndim()
+        dim = self.getndim()
         fmt = self.getformat()
         if dim == 0:
             raise NotImplementedError
         elif dim == 1:
-            return self._tolist(space, buf, self.getlength(), fmt)
+            itemsize = self.getitemsize()
+            return self._tolist(space, buf, buf.getlength() // itemsize, fmt)
         else:
-            return self._tolist_rec(space, buf, fmt)
+            return self._tolist_rec(space, buf, 0, 0, fmt)
 
     def _tolist(self, space, buf, count, fmt):
         # TODO: this probably isn't very fast
-        fmtiter = UnpackFormatIterator(space, self.buf)
+        fmtiter = UnpackFormatIterator(space, buf)
         fmtiter.interpret(fmt * count)
         return space.newlist(fmtiter.result_w)
 
-    def _tolist_rec(self, space, buf, start, dim, fmt):
-        idim = dim-1
+    def _tolist_rec(self, space, buf, start, idim, fmt):
         strides = self.getstrides()
+        shape = self.getshape()
+        #
+        dim = idim+1
         stride = strides[idim]
         itemsize = self.getitemsize()
-        if dim >= buf.getndim():
-            return self._tolist(space, SubBuffer(buf, start, itemsize), stride // itemsize, fmt)
-        shape = self.getshape()
         dimshape = shape[idim]
+        #
+        if dim >= self.getndim():
+            bytecount = (stride * dimshape)
+            count = bytecount // itemsize
+            return self._tolist(space, SubBuffer(buf, start, bytecount), count, fmt)
         items = [None] * dimshape
 
         for i in range(dimshape):
-            item = self._tolist_rec(space, SubBuffer(buf, start, self.itemsize), dim+1, fmt)
+            bytecount = stride
+            item = self._tolist_rec(space, SubBuffer(buf, start, bytecount), start, idim+1, fmt)
             items[i] = item
             start += stride
 
-        return space.newlist(items,len(items))
+        return space.newlist(items)
 
 
     def _start_from_tuple(self, space, w_tuple):
@@ -433,7 +439,8 @@
         origfmt = mv.getformat()
         mv._cast_to_1D(space, origfmt, fmt, itemsize)
         if w_shape:
-            shape = [space.int_w(w_obj) for w_obj in w_shape.getitems_unroll()]
+            fview = space.fixedview(w_shape)
+            shape = [space.int_w(w_obj) for w_obj in fview]
             mv._cast_to_ND(space, shape, ndim)
         return mv
 
@@ -536,12 +543,15 @@
         self._init_flags()
 
     def _init_strides_from_shape(self):
-        s = [0] * len(self.shape)
+        shape = self.getshape()
+        s = [0] * len(shape)
         self.strides = s
-        dim = self.getndim()
-        s[dim-1] = self.itemsize
-        for i in range(0,ndim-2,-1):
+        ndim = self.getndim()
+        s[ndim-1] = self.itemsize
+        i = ndim-2
+        while i >= 0:
             s[i] = s[i+1] * shape[i+1]
+            i -= 1
 
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -208,14 +208,16 @@
         items = []
         if size == 0:
             return ''
-        bytecount = (stop - start)
-        # data is stores as list of ints, thus this gets around the
-        # issue that one cannot advance in bytes
-        count = bytecount // size
-        start = start // size
-        for i in range(start, start+count, step):
-            items.append(self.getitem(i))
-        return ''.join(items)
+        return ''.join([self.getitem(i) for i in range(start,stop,step)])
+        #bytecount = (stop - start)
+        ## data is stores as list of ints, thus this gets around the
+        ## issue that one cannot advance in bytes
+        #count = bytecount // size
+        #start = start // size
+        #for i in range(start, start+count, step):
+        #    items.append(self.getitem(i))
+        #return ''.join(items)
+
 
     def getformat(self):
         return self.format
@@ -338,8 +340,10 @@
                     strides=[8], shape=[6])
         view = memoryview(empty)
         byteview = view.cast('b')
-        assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]
+        #assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]
         i32view = byteview.cast('i', shape=[1,3])
         assert i32view.format == 'i'
         assert i32view.itemsize == 4
         assert i32view.tolist() == [[1,2,3]]
+        i32view = byteview.cast('i', shape=(1,3))
+        assert i32view.tolist() == [[1,2,3]]

From pypy.commits at gmail.com  Mon Aug 29 06:17:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 03:17:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Finally found and fixed the
 UnicodeDecodeError escaping in RPython
Message-ID: <57c40ba0.09afc20a.8df27.0b6f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86681:7528d42874ea
Date: 2016-08-29 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/7528d42874ea/

Log:	Finally found and fixed the UnicodeDecodeError escaping in RPython

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -838,11 +838,16 @@
         return w_s1
 
     def get_interned_str(self, s):
-        """Assumes an identifier (utf-8 encoded str)"""
+        """Assumes an identifier (utf-8 encoded str).  Returns None if
+        the identifier is not interned, or not a valid utf-8 string at all.
+        """
         # interface for marshal_impl
         if not we_are_translated():
             assert type(s) is str
-        u = s.decode('utf-8')
+        try:
+            u = s.decode('utf-8')
+        except UnicodeDecodeError:
+            return None
         return self.interned_strings.get(u)   # may be None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -459,7 +459,8 @@
 
 @unmarshaller(TYPE_INTERNED)
 def unmarshal_bytes(space, u, tc):
-    return space.new_interned_str(u.get_str())
+    w_u = unmarshal_unicode(space, u, tc)
+    return u.space.new_interned_w_str(w_u)
 
 def _unmarshal_ascii(u, short_length, interned):
     if short_length:

From pypy.commits at gmail.com  Mon Aug 29 06:21:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 03:21:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <57c40cc4.e2efc20a.ff32c.0213@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86682:baa0b1b60dee
Date: 2016-08-29 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/baa0b1b60dee/

Log:	fix test

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -16,12 +16,14 @@
     return codegen.compile_ast(space, ast, info)
 
 def generate_function_code(expr, space):
+    from pypy.interpreter.astcompiler.ast import FunctionDef
     p = pyparse.PythonParser(space)
     info = pyparse.CompileInfo("<test>", 'exec')
     cst = p.parse_source(expr, info)
     ast = astbuilder.ast_from_node(space, cst, info)
     function_ast = optimize.optimize_ast(space, ast.body[0], info)
     function_ast = ast.body[0]
+    assert isinstance(function_ast, FunctionDef)
     symbols = symtable.SymtableBuilder(space, ast, info)
     generator = codegen.FunctionCodeGenerator(
         space, 'function', function_ast, 1, symbols, info, qualname='function')
@@ -1346,7 +1348,9 @@
             assert ops.BINARY_POWER not in counts
 
     def test_call_function_var(self):
-        source = """call(*me)"""
+        source = """def f():
+            call(*me)
+        """
         code, blocks = generate_function_code(source, self.space)
         # there is a stack computation error
         assert blocks[0].instructions[3].arg == 0

From pypy.commits at gmail.com  Mon Aug 29 06:42:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 03:42:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Improve testing of validate.py. Fix it.
Message-ID: <57c41198.02c41c0a.7d8c2.4fd3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86683:8e7d1dc6fc7e
Date: 2016-08-29 12:41 +0200
http://bitbucket.org/pypy/pypy/changeset/8e7d1dc6fc7e/

Log:	Improve testing of validate.py. Fix it.

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -4,12 +4,20 @@
 from pypy.interpreter.pyparser.pygram import syms, tokens
 from pypy.interpreter.pyparser.error import SyntaxError
 from pypy.interpreter.pyparser import parsestring
-from rpython.rlib.objectmodel import always_inline
+from rpython.rlib.objectmodel import always_inline, we_are_translated
 
 
 def ast_from_node(space, node, compile_info):
     """Turn a parse tree, node, to AST."""
-    return ASTBuilder(space, node, compile_info).build_ast()
+    ast = ASTBuilder(space, node, compile_info).build_ast()
+    #
+    # When we are not translated, we send this ast to validate_ast.
+    # The goal is to check that validate_ast doesn't crash on valid
+    # asts, at least.
+    if not we_are_translated():
+        from pypy.interpreter.astcompiler import validate
+        validate.validate_ast(space, ast)
+    return ast
 
 
 augassign_operator_map = {
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -176,6 +176,13 @@
         if node.returns:
             self._validate_expr(node.returns)
 
+    def visit_AsyncFunctionDef(self, node):
+        self._validate_body(node.body, "AsyncFunctionDef")
+        node.args.walkabout(self)
+        self._validate_exprs(node.decorator_list)
+        if node.returns:
+            self._validate_expr(node.returns)
+
     def visit_keyword(self, node):
         self._validate_expr(node.value)
 
@@ -193,6 +200,9 @@
         if node.value:
             self._validate_expr(node.value)
 
+    def visit_Await(self, node):
+        self._validate_expr(node.value)
+
     def visit_Delete(self, node):
         self._validate_nonempty_seq(node.targets, "targets", "Delete")
         self._validate_exprs(node.targets, ast.Del)
@@ -212,6 +222,12 @@
         self._validate_body(node.body, "For")
         self._validate_stmts(node.orelse)
 
+    def visit_AsyncFor(self, node):
+        self._validate_expr(node.target, ast.Store)
+        self._validate_expr(node.iter)
+        self._validate_body(node.body, "AsyncFor")
+        self._validate_stmts(node.orelse)
+
     def visit_While(self, node):
         self._validate_expr(node.test)
         self._validate_body(node.body, "While")
@@ -232,6 +248,11 @@
         self.visit_sequence(node.items)
         self._validate_body(node.body, "With")
 
+    def visit_AsyncWith(self, node):
+        self._validate_nonempty_seq(node.items, "items", "AsyncWith")
+        self.visit_sequence(node.items)
+        self._validate_body(node.body, "AsyncWith")
+
     def visit_Raise(self, node):
         if node.exc:
             self._validate_expr(node.exc)

From pypy.commits at gmail.com  Mon Aug 29 07:03:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 04:03:28 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: I'll pick pathlib next
Message-ID: <57c41680.04141c0a.2e312.51e6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5689:fe32ac59353e
Date: 2016-08-29 13:03 +0200
http://bitbucket.org/pypy/extradoc/changeset/fe32ac59353e/

Log:	I'll pick pathlib next

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -11,6 +11,8 @@
   We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
   Matti's opinion on that would be great!
 
+* arigo: pathlib: Object-oriented filesystem paths (PEP 428)
+
 
 Finished
 --------

From pypy.commits at gmail.com  Mon Aug 29 07:05:10 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 04:05:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: merge py3.5
Message-ID: <57c416e6.54bc1c0a.7f9a9.549c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86684:493de2daaa30
Date: 2016-08-29 13:04 +0200
http://bitbucket.org/pypy/pypy/changeset/493de2daaa30/

Log:	merge py3.5

diff too long, truncating to 2000 out of 4771 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
diff --git a/lib-python/3/importlib/_bootstrap_external.py b/lib-python/3/importlib/_bootstrap_external.py
--- a/lib-python/3/importlib/_bootstrap_external.py
+++ b/lib-python/3/importlib/_bootstrap_external.py
@@ -228,7 +228,14 @@
 # longer be understood by older implementations of the eval loop (usually
 # due to the addition of new opcodes).
 
-MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n'
+# MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n'
+#
+# PyPy change: the MAGIC_NUMBER is defined in
+# pypy/interpreter/pycode.py, 'default_magic'.  It is based on a number
+# different than CPython's, always < 3000.  We get the 4-bytes string
+# here via a hack: MAGIC_NUMBER is set in the module from
+# module/_frozen_importlib/__init__.py before the module is executed.
+
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -554,6 +554,9 @@
     def putwin(self, filep):
         # filestar = ffi.new("FILE *", filep)
         return _check_ERR(lib.putwin(self._win, filep), "putwin")
+        # XXX CPython 3.5 says: We have to simulate this by writing to
+        # a temporary FILE*, then reading back, then writing to the
+        # argument stream.
 
     def redrawln(self, beg, num):
         return _check_ERR(lib.wredrawln(self._win, beg, num), "redrawln")
@@ -704,6 +707,7 @@
 
 
 def getwin(filep):
+    # XXX CPython 3.5: there's logic to use a temp file instead
     return Window(_check_NULL(lib.getwin(filep)))
 
 
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,219 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. 
+
+XXXXX MORE ???
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Support for memoryview attributes (format, itemsize, ...) which also
+    adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,157 +1,8 @@
-=========================
-What's new in PyPy2.7 5.3+
-=========================
+==========================
+What's new in PyPy2.7 5.4+
+==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
 
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
-
-
-.. branch: const-fold-we-are-jitted
-
-Reduce the size of the generated C code by constant-folding ``we_are_jitted``
-in non-jitcode.
-
-.. branch: memoryview-attributes
-
-Support for memoryview attributes (format, itemsize, ...).
-Extends the cpyext emulation layer.
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -0,0 +1,165 @@
+=========================
+What's new in PyPy2.7 5.4
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
+
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -432,7 +432,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(w_decorator_list)
         _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        _returns = expr.from_object(space, w_returns) if w_returns is not None else None
+        _returns = expr.from_object(space, w_returns)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return FunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset)
@@ -508,7 +508,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(w_decorator_list)
         _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        _returns = expr.from_object(space, w_returns) if w_returns is not None else None
+        _returns = expr.from_object(space, w_returns)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return AsyncFunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset)
@@ -630,7 +630,7 @@
         w_value = get_field(space, w_node, 'value', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _value = expr.from_object(space, w_value) if w_value is not None else None
+        _value = expr.from_object(space, w_value)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Return(_value, _lineno, _col_offset)
@@ -1190,8 +1190,8 @@
         w_cause = get_field(space, w_node, 'cause', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _exc = expr.from_object(space, w_exc) if w_exc is not None else None
-        _cause = expr.from_object(space, w_cause) if w_cause is not None else None
+        _exc = expr.from_object(space, w_exc)
+        _cause = expr.from_object(space, w_cause)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Raise(_exc, _cause, _lineno, _col_offset)
@@ -1314,7 +1314,7 @@
         _test = expr.from_object(space, w_test)
         if _test is None:
             raise_required_value(space, w_node, 'test')
-        _msg = expr.from_object(space, w_msg) if w_msg is not None else None
+        _msg = expr.from_object(space, w_msg)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Assert(_test, _msg, _lineno, _col_offset)
@@ -2312,7 +2312,7 @@
         w_value = get_field(space, w_node, 'value', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _value = expr.from_object(space, w_value) if w_value is not None else None
+        _value = expr.from_object(space, w_value)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Yield(_value, _lineno, _col_offset)
@@ -3101,9 +3101,9 @@
         w_lower = get_field(space, w_node, 'lower', True)
         w_upper = get_field(space, w_node, 'upper', True)
         w_step = get_field(space, w_node, 'step', True)
-        _lower = expr.from_object(space, w_lower) if w_lower is not None else None
-        _upper = expr.from_object(space, w_upper) if w_upper is not None else None
-        _step = expr.from_object(space, w_step) if w_step is not None else None
+        _lower = expr.from_object(space, w_lower)
+        _upper = expr.from_object(space, w_upper)
+        _step = expr.from_object(space, w_step)
         return Slice(_lower, _upper, _step)
 
 State.ast_type('Slice', 'slice', ['lower', 'upper', 'step'])
@@ -3583,7 +3583,7 @@
         w_body = get_field(space, w_node, 'body', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _type = expr.from_object(space, w_type) if w_type is not None else None
+        _type = expr.from_object(space, w_type)
         _name = space.str_or_None_w(w_name)
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
@@ -3664,12 +3664,12 @@
         w_defaults = get_field(space, w_node, 'defaults', False)
         args_w = space.unpackiterable(w_args)
         _args = [arg.from_object(space, w_item) for w_item in args_w]
-        _vararg = arg.from_object(space, w_vararg) if w_vararg is not None else None
+        _vararg = arg.from_object(space, w_vararg) if not space.is_w(w_vararg, space.w_None) else None
         kwonlyargs_w = space.unpackiterable(w_kwonlyargs)
         _kwonlyargs = [arg.from_object(space, w_item) for w_item in kwonlyargs_w]
         kw_defaults_w = space.unpackiterable(w_kw_defaults)
         _kw_defaults = [expr.from_object(space, w_item) for w_item in kw_defaults_w]
-        _kwarg = arg.from_object(space, w_kwarg) if w_kwarg is not None else None
+        _kwarg = arg.from_object(space, w_kwarg) if not space.is_w(w_kwarg, space.w_None) else None
         defaults_w = space.unpackiterable(w_defaults)
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwonlyargs, _kw_defaults, _kwarg, _defaults)
@@ -3705,7 +3705,7 @@
         _arg = space.identifier_w(w_arg)
         if _arg is None:
             raise_required_value(space, w_node, 'arg')
-        _annotation = expr.from_object(space, w_annotation) if w_annotation is not None else None
+        _annotation = expr.from_object(space, w_annotation)
         return arg(_arg, _annotation)
 
 State.ast_type('arg', 'AST', ['arg', 'annotation'])
@@ -3805,7 +3805,7 @@
         _context_expr = expr.from_object(space, w_context_expr)
         if _context_expr is None:
             raise_required_value(space, w_node, 'context_expr')
-        _optional_vars = expr.from_object(space, w_optional_vars) if w_optional_vars is not None else None
+        _optional_vars = expr.from_object(space, w_optional_vars)
         return withitem(_context_expr, _optional_vars)
 
 State.ast_type('withitem', 'AST', ['context_expr', 'optional_vars'])
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -4,12 +4,20 @@
 from pypy.interpreter.pyparser.pygram import syms, tokens
 from pypy.interpreter.pyparser.error import SyntaxError
 from pypy.interpreter.pyparser import parsestring
-from rpython.rlib.objectmodel import always_inline
+from rpython.rlib.objectmodel import always_inline, we_are_translated
 
 
 def ast_from_node(space, node, compile_info):
     """Turn a parse tree, node, to AST."""
-    return ASTBuilder(space, node, compile_info).build_ast()
+    ast = ASTBuilder(space, node, compile_info).build_ast()
+    #
+    # When we are not translated, we send this ast to validate_ast.
+    # The goal is to check that validate_ast doesn't crash on valid
+    # asts, at least.
+    if not we_are_translated():
+        from pypy.interpreter.astcompiler import validate
+        validate.validate_ast(space, ast)
+    return ast
 
 
 augassign_operator_map = {
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1234,10 +1234,7 @@
         if d.values:
             for i in range(len(d.values)):
                 key = d.keys[i]
-                if key is None:
-                    is_unpacking = True
-                else:
-                    is_unpacking = False
+                is_unpacking = key is None
                 if elements == 0xFFFF or (elements and is_unpacking):
                     self.emit_op_arg(ops.BUILD_MAP, elements)
                     containers += 1
@@ -1265,7 +1262,7 @@
             oparg = min(containers, 255)
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
-            is_unpacking = 0
+            is_unpacking = False
     
     def visit_Set(self, s):
         self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -16,12 +16,14 @@
     return codegen.compile_ast(space, ast, info)
 
 def generate_function_code(expr, space):
+    from pypy.interpreter.astcompiler.ast import FunctionDef
     p = pyparse.PythonParser(space)
     info = pyparse.CompileInfo("<test>", 'exec')
     cst = p.parse_source(expr, info)
     ast = astbuilder.ast_from_node(space, cst, info)
     function_ast = optimize.optimize_ast(space, ast.body[0], info)
     function_ast = ast.body[0]
+    assert isinstance(function_ast, FunctionDef)
     symbols = symtable.SymtableBuilder(space, ast, info)
     generator = codegen.FunctionCodeGenerator(
         space, 'function', function_ast, 1, symbols, info, qualname='function')
@@ -864,9 +866,10 @@
         with a: pass
         with a: pass
         with a: pass
+        with a: pass
         """
         code = compile_with_astcompiler(source, 'exec', self.space)
-        assert code.co_stacksize == 5
+        assert code.co_stacksize == 6  # i.e. <= 7, there is no systematic leak
 
     def test_stackeffect_bug5(self):
         source = """if 1:
@@ -1345,7 +1348,9 @@
             assert ops.BINARY_POWER not in counts
 
     def test_call_function_var(self):
-        source = """call(*me)"""
+        source = """def f():
+            call(*me)
+        """
         code, blocks = generate_function_code(source, self.space)
         # there is a stack computation error
         assert blocks[0].instructions[3].arg == 0
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -160,7 +160,17 @@
         else:
             extractor = "%s.from_object(space, %s)" % (field.type, value)
             if field.opt:
-                extractor += " if %s is not None else None" % (value,)
+                if field.type == 'expr':
+                    # the expr.from_object() method should accept w_None and
+                    # return None; nothing more to do here
+                    pass
+                elif field.type == 'arg':
+                    # the method arg.from_object() doesn't accept w_None
+                    extractor += (
+                        ' if not space.is_w(%s, space.w_None) else None'
+                        % (value,))
+                else:
+                    raise NotImplementedError(field.type)
             return extractor
 
     def get_field_converter(self, field):
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -176,6 +176,13 @@
         if node.returns:
             self._validate_expr(node.returns)
 
+    def visit_AsyncFunctionDef(self, node):
+        self._validate_body(node.body, "AsyncFunctionDef")
+        node.args.walkabout(self)
+        self._validate_exprs(node.decorator_list)
+        if node.returns:
+            self._validate_expr(node.returns)
+
     def visit_keyword(self, node):
         self._validate_expr(node.value)
 
@@ -193,6 +200,9 @@
         if node.value:
             self._validate_expr(node.value)
 
+    def visit_Await(self, node):
+        self._validate_expr(node.value)
+
     def visit_Delete(self, node):
         self._validate_nonempty_seq(node.targets, "targets", "Delete")
         self._validate_exprs(node.targets, ast.Del)
@@ -212,6 +222,12 @@
         self._validate_body(node.body, "For")
         self._validate_stmts(node.orelse)
 
+    def visit_AsyncFor(self, node):
+        self._validate_expr(node.target, ast.Store)
+        self._validate_expr(node.iter)
+        self._validate_body(node.body, "AsyncFor")
+        self._validate_stmts(node.orelse)
+
     def visit_While(self, node):
         self._validate_expr(node.test)
         self._validate_body(node.body, "While")
@@ -232,6 +248,11 @@
         self.visit_sequence(node.items)
         self._validate_body(node.body, "With")
 
+    def visit_AsyncWith(self, node):
+        self._validate_nonempty_seq(node.items, "items", "AsyncWith")
+        self.visit_sequence(node.items)
+        self._validate_body(node.body, "AsyncWith")
+
     def visit_Raise(self, node):
         if node.exc:
             self._validate_expr(node.exc)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -837,13 +837,18 @@
             self.interned_strings.set(u, w_s1)
         return w_s1
 
-    def is_interned_str(self, s):
-        """Assumes an identifier (utf-8 encoded str)"""
+    def get_interned_str(self, s):
+        """Assumes an identifier (utf-8 encoded str).  Returns None if
+        the identifier is not interned, or not a valid utf-8 string at all.
+        """
         # interface for marshal_impl
         if not we_are_translated():
             assert type(s) is str
-        u = s.decode('utf-8')
-        return self.interned_strings.get(u) is not None
+        try:
+            u = s.decode('utf-8')
+        except UnicodeDecodeError:
+            return None
+        return self.interned_strings.get(u)   # may be None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -35,9 +35,10 @@
 
 # we compute the magic number in a similar way to CPython, but we use a
 # different value for the highest 16 bits. Bump pypy_incremental_magic every
-# time you make pyc files incompatible
+# time you make pyc files incompatible.  This value ends up in the frozen
+# importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 64 # bump it by 16
+pypy_incremental_magic = 80 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1198,7 +1198,7 @@
                 self.settopvalue(self.space.w_None)
 
     @jit.unroll_safe
-    def call_function(self, oparg, w_star=None, w_starstar=None):
+    def call_function(self, oparg, w_starstar=None, has_vararg=False):
         n_arguments = oparg & 0xff
         n_keywords = (oparg>>8) & 0xff
         if n_keywords:
@@ -1210,20 +1210,16 @@
                     break
                 w_value = self.popvalue()
                 w_key = self.popvalue()
-                # temporary (dirty) fix: if star-arg occurs after kwarg,
-                # arg order is reversed on stack
-                from pypy.objspace.std.listobject import W_ListObject
-                if isinstance(w_key, W_ListObject):
-                    w_key_temp = w_key
-                    w_key = w_value
-                    w_value = w_star
-                    w_star = w_key_temp
                 key = self.space.identifier_w(w_key)
                 keywords[n_keywords] = key
                 keywords_w[n_keywords] = w_value
         else:
             keywords = None
             keywords_w = None
+        if has_vararg:
+            w_star = self.popvalue()
+        else:
+            w_star = None
         arguments = self.popvalues(n_arguments)
         args = self.argument_factory(arguments, keywords, keywords_w, w_star,
                                      w_starstar)
@@ -1252,17 +1248,15 @@
             self.call_function(oparg)
 
     def CALL_FUNCTION_VAR(self, oparg, next_instr):
-        w_varargs = self.popvalue()
-        self.call_function(oparg, w_varargs)
+        self.call_function(oparg, has_vararg=True)
 
     def CALL_FUNCTION_KW(self, oparg, next_instr):
         w_varkw = self.popvalue()
-        self.call_function(oparg, None, w_varkw)
+        self.call_function(oparg, w_varkw)
 
     def CALL_FUNCTION_VAR_KW(self, oparg, next_instr):
         w_varkw = self.popvalue()
-        w_varargs = self.popvalue()
-        self.call_function(oparg, w_varargs, w_varkw)
+        self.call_function(oparg, w_varkw, has_vararg=True)
 
     @jit.unroll_safe
     def _make_function(self, oparg, freevars=None):
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -214,6 +214,15 @@
         assert self.codetest(code, 'g', [12, {}]) ==    ()
         assert self.codetest(code, 'g', [12, {3:1}]) == (3,)
 
+    def test_star_arg_after_keyword_arg(self):
+        code = '''
+            def f(a, b):
+                return a - b
+            def g(a, b):
+                return f(b=b, *(a,))
+        '''
+        assert self.codetest(code, 'g', [40, 2]) == 38
+
     def test_closure(self):
         code = '''
             def f(x, y):
diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py
deleted file mode 100644
--- a/pypy/module/_file/readinto.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import sys, errno
-from rpython.rlib import rposix
-from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._file.interp_file import is_wouldblock_error, signal_checker
-
-_WIN32 = sys.platform.startswith('win')
-UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
-
-os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read',
-                          [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
-                          rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO)
-
-
-def direct_readinto(self, w_rwbuffer):
-    rwbuffer = self.space.writebuf_w(w_rwbuffer)
-    stream = self.getstream()
-    size = rwbuffer.getlength()
-    target_address = lltype.nullptr(rffi.CCHARP.TO)
-    fd = -1
-    target_pos = 0
-
-    if size > 64:
-        try:
-            target_address = rwbuffer.get_raw_address()
-        except ValueError:
-            pass
-        else:
-            fd = stream.try_to_find_file_descriptor()
-
-    if fd < 0 or not target_address:
-        # fall-back
-        MAX_PART = 1024 * 1024    # 1 MB
-        while size > MAX_PART:
-            data = self.direct_read(MAX_PART)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-            if len(data) != MAX_PART:
-                break
-        else:
-            data = self.direct_read(size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-
-    else:
-        # optimized case: reading more than 64 bytes into a rwbuffer
-        # with a valid raw address
-        self.check_readable()
-
-        # first "read" the part that is already sitting in buffers, if any
-        initial_size = min(size, stream.count_buffered_bytes())
-        if initial_size > 0:
-            data = stream.read(initial_size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-
-        # then call os_read() to get the rest
-        if size > 0:
-            stream.flush()
-            while True:
-                got = os_read(fd, rffi.ptradd(target_address, target_pos), size)
-                got = rffi.cast(lltype.Signed, got)
-                if got > 0:
-                    target_pos += got
-                    size -= got
-                    if size <= 0:
-                        break
-                elif got == 0:
-                    break
-                else:
-                    err = rposix.get_saved_errno()
-                    if err == errno.EINTR:
-                        signal_checker(self.space)()
-                        continue
-                    if is_wouldblock_error(err) and target_pos > 0:
-                        break
-                    raise OSError(err, "read error")
-            keepalive_until_here(rwbuffer)
-
-    return self.space.wrap(target_pos)
diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -19,7 +19,8 @@
         with open(os.path.join(lib_python, 'importlib', name + '.py')) as fp:
             source = fp.read()
         pathname = "<frozen importlib.%s>" % name
-        code_w = Module._cached_compile(space, source, pathname, 'exec', 0)
+        code_w = Module._cached_compile(space, name, source,
+                                        pathname, 'exec', 0)
         space.setitem(w_dict, space.wrap('__name__'), w_name)
         space.setitem(w_dict, space.wrap('__builtins__'),
                       space.wrap(space.builtin))
@@ -27,10 +28,15 @@
 
     def install(self):
         """NOT_RPYTHON"""
+        from pypy.module.imp import interp_imp
+
         super(Module, self).install()
         space = self.space
         # "import importlib/_boostrap_external.py"
         w_mod = Module(space, space.wrap("_frozen_importlib_external"))
+        # hack: inject MAGIC_NUMBER into this module's dict
+        space.setattr(w_mod, space.wrap('MAGIC_NUMBER'),
+                      interp_imp.get_magic(space))
         self._compile_bootstrap_module(
             space, '_bootstrap_external', w_mod.w_name, w_mod.w_dict)
         space.sys.setmodule(w_mod)
@@ -43,11 +49,13 @@
         self.w_import = space.wrap(interp_import.import_with_frames_removed)
 
     @staticmethod
-    def _cached_compile(space, source, *args):
+    def _cached_compile(space, name, source, *args):
         from rpython.config.translationoption import CACHE_DIR
         from pypy.module.marshal import interp_marshal
+        from pypy.interpreter.pycode import default_magic
 
-        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_bootstrap')
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_%d%s' % (
+            default_magic, name))
         try:
             if space.config.translating:
                 raise IOError("don't use the cache when translating pypy")
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -4,6 +4,7 @@
     OperationError, oefmt, wrap_oserror, wrap_oserror2)
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import rposix
 from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC, O_EXCL
 import sys, os, stat, errno
 from pypy.module._io.interp_iobase import W_RawIOBase, convert_size
@@ -29,6 +30,7 @@
 
 O_BINARY = getattr(os, "O_BINARY", 0)
 O_APPEND = getattr(os, "O_APPEND", 0)
+_open_inhcache = rposix.SetNonInheritableCache()
 
 def _bad_mode(space):
     raise oefmt(space.w_ValueError,
@@ -139,6 +141,7 @@
 
     @unwrap_spec(mode=str, closefd=int)
     def descr_init(self, space, w_name, mode='r', closefd=True, w_opener=None):
+        self._close(space)
         if space.isinstance_w(w_name, space.w_float):
             raise oefmt(space.w_TypeError,
                         "integer argument expected, got float")
@@ -153,6 +156,8 @@
                 raise oefmt(space.w_ValueError, "negative file descriptor")
 
         self.readable, self.writable, self.created, self.appending, flags = decode_mode(space, mode)
+        if rposix.O_CLOEXEC is not None:
+            flags |= rposix.O_CLOEXEC
 
         fd_is_own = False
         try:
@@ -171,8 +176,7 @@
                     raise oefmt(space.w_ValueError,
                                 "Cannot use closefd=False with file name")
 
-                from pypy.module.posix.interp_posix import (
-                    dispatch_filename, rposix)
+                from pypy.module.posix.interp_posix import dispatch_filename
                 try:
                     self.fd = dispatch_filename(rposix.open)(
                         space, w_name, flags, 0666)
@@ -181,6 +185,11 @@
                                         exception_name='w_IOError')
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    try:
+                        _open_inhcache.set_non_inheritable(self.fd)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
             else:
                 w_fd = space.call_function(w_opener, w_name, space.wrap(flags))
                 try:
@@ -192,6 +201,11 @@
                                 "expected integer from opener")
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    try:
+                        rposix.set_inheritable(self.fd, False)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
 
             self._dircheck(space, w_name)
             space.setattr(self, space.wrap("name"), w_name)
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -246,6 +246,33 @@
             assert f.mode == 'xb'
         raises(FileExistsError, _io.FileIO, filename, 'x')
 
+    def test_non_inheritable(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        assert posix.get_inheritable(f.fileno()) == False
+        f.close()
+
+    def test_FileIO_fd_does_not_change_inheritable(self):
+        import _io, posix
+        fd1, fd2 = posix.pipe()
+        posix.set_inheritable(fd1, True)
+        posix.set_inheritable(fd2, False)
+        f1 = _io.FileIO(fd1, 'r')
+        f2 = _io.FileIO(fd2, 'w')
+        assert posix.get_inheritable(fd1) == True
+        assert posix.get_inheritable(fd2) == False
+        f1.close()
+        f2.close()
+
+    def test_close_upon_reinit(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        fd1 = f.fileno()
+        f.__init__(self.tmpfile, 'w')
+        fd2 = f.fileno()
+        if fd1 != fd2:
+            raises(OSError, posix.close, fd1)
+
 
 def test_flush_at_exit():
     from pypy import conftest
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.c b/pypy/module/_posixsubprocess/_posixsubprocess.c
--- a/pypy/module/_posixsubprocess/_posixsubprocess.c
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.c
@@ -106,6 +106,30 @@
 }
 
 
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable);   /* rposix.py */
+
+static int
+make_inheritable(long *py_fds_to_keep, ssize_t num_fds_to_keep,
+                 int errpipe_write)
+{
+    long i;
+
+    for (i = 0; i < num_fds_to_keep; ++i) {
+        long fd = py_fds_to_keep[i];
+        if (fd == errpipe_write) {
+            /* errpipe_write is part of py_fds_to_keep. It must be closed at
+               exec(), but kept open in the child process until exec() is
+               called. */
+            continue;
+        }
+        if (rpy_set_inheritable((int)fd, 1) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+
 /* Close all file descriptors in the range start_fd inclusive to
  * end_fd exclusive except for those in py_fds_to_keep.  If the
  * range defined by [start_fd, end_fd) is large this will take a
@@ -329,6 +353,9 @@
     /* Buffer large enough to hold a hex integer.  We can't malloc. */
     char hex_errno[sizeof(saved_errno)*2+1];
 
+    if (make_inheritable(py_fds_to_keep, num_fds_to_keep, errpipe_write) < 0)
+        goto error;
+
     /* Close parent's pipe ends. */
     if (p2cwrite != -1) {
         POSIX_CALL(close(p2cwrite));
@@ -352,26 +379,25 @@
        dup2() removes the CLOEXEC flag but we must do it ourselves if dup2()
        would be a no-op (issue #10806). */
     if (p2cread == 0) {
-        int old = fcntl(p2cread, F_GETFD);
-        if (old != -1)
-            fcntl(p2cread, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (p2cread != -1) {
+        if (rpy_set_inheritable(p2cread, 1) < 0)
+            goto error;
+    }
+    else if (p2cread != -1)
         POSIX_CALL(dup2(p2cread, 0));  /* stdin */
+
+    if (c2pwrite == 1) {
+        if (rpy_set_inheritable(c2pwrite, 1) < 0)
+            goto error;
     }
-    if (c2pwrite == 1) {
-        int old = fcntl(c2pwrite, F_GETFD);
-        if (old != -1)
-            fcntl(c2pwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (c2pwrite != -1) {
+    else if (c2pwrite != -1)
         POSIX_CALL(dup2(c2pwrite, 1));  /* stdout */
+
+    if (errwrite == 2) {
+        if (rpy_set_inheritable(errwrite, 1) < 0)
+            goto error;
     }
-    if (errwrite == 2) {
-        int old = fcntl(errwrite, F_GETFD);
-        if (old != -1)
-            fcntl(errwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (errwrite != -1) {
+    else if (errwrite != -1)
         POSIX_CALL(dup2(errwrite, 2));  /* stderr */
-    }
 
     /* Close pipe fds.  Make sure we don't close the same fd more than */
     /* once, or standard fds. */
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.h b/pypy/module/_posixsubprocess/_posixsubprocess.h
--- a/pypy/module/_posixsubprocess/_posixsubprocess.h
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.h
@@ -1,3 +1,4 @@
+#include <unistd.h>    /* for ssize_t */
 #include "src/precommondefs.h"
 
 RPY_EXTERN void
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -5,6 +5,7 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib import rposix
 
 from pypy.interpreter.error import (
     OperationError, exception_from_saved_errno, oefmt, wrap_oserror)
@@ -36,6 +37,7 @@
     compile_extra.append("-DHAVE_SETSID")
 
 eci = eci.merge(
+    rposix.eci_inheritable,
     ExternalCompilationInfo(
         compile_extra=compile_extra))
 
diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py
--- a/pypy/module/_posixsubprocess/test/test_subprocess.py
+++ b/pypy/module/_posixsubprocess/test/test_subprocess.py
@@ -75,3 +75,18 @@
         n = 1
         raises(OverflowError, _posixsubprocess.fork_exec,
                1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17)
+
+    def test_pass_fds_make_inheritable(self):
+        import subprocess, posix
+
+        fd1, fd2 = posix.pipe()
+        assert posix.get_inheritable(fd1) is False
+        assert posix.get_inheritable(fd2) is False
+
+        subprocess.check_call(['/usr/bin/env', 'python', '-c',
+                               'import os;os.write(%d,b"K")' % fd2],
+                              close_fds=True, pass_fds=[fd2])
+        res = posix.read(fd1, 1)
+        assert res == b"K"
+        posix.close(fd1)
+        posix.close(fd2)
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -142,21 +142,11 @@
 
 @unwrap_spec(fd=int)
 def dup(space, fd):
-    newfd = rsocket.dup(fd)
-    return space.wrap(newfd)
-
- at unwrap_spec(fd=int, family=int, type=int, proto=int)
-def fromfd(space, fd, family, type, proto=0):
-    """fromfd(fd, family, type[, proto]) -> socket object
-
-    Create a socket object from the given file descriptor.
-    The remaining arguments are the same as for socket().
-    """
     try:
-        sock = rsocket.fromfd(fd, family, type, proto)
+        newfd = rsocket.dup(fd, inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(W_Socket(space, sock))
+    return space.wrap(newfd)
 
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
@@ -170,7 +160,8 @@
     AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
     """
     try:
-        sock1, sock2 = rsocket.socketpair(family, type, proto)
+        sock1, sock2 = rsocket.socketpair(family, type, proto,
+                                          inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
     return space.newtuple([
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -177,7 +177,7 @@
                 sock = RSocket(family, type, proto,
                                fd=space.c_filedescriptor_w(w_fileno))
             else:
-                sock = RSocket(family, type, proto)
+                sock = RSocket(family, type, proto, inheritable=False)
             W_Socket.__init__(self, space, sock)
         except SocketError as e:
             raise converted_error(space, e)
@@ -228,7 +228,7 @@
         For IP sockets, the address info is a pair (hostaddr, port).
         """
         try:
-            fd, addr = self.sock.accept()
+            fd, addr = self.sock.accept(inheritable=False)
             return space.newtuple([space.wrap(fd),
                                    addr_as_object(addr, fd, space)])
         except SocketError as e:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -546,11 +546,19 @@
         s.ioctl(_socket.SIO_KEEPALIVE_VALS, (1, 100, 100))
 
     def test_dup(self):
-        import _socket as socket
+        import _socket as socket, posix
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.bind(('localhost', 0))
         fd = socket.dup(s.fileno())
         assert s.fileno() != fd
+        assert posix.get_inheritable(s.fileno()) is False
+        assert posix.get_inheritable(fd) is False
+        posix.close(fd)
+        s.close()
+
+    def test_dup_error(self):
+        import _socket
+        raises(_socket.error, _socket.dup, 123456)
 
     def test_buffer(self):
         # Test that send/sendall/sendto accept a buffer as arg
@@ -648,6 +656,26 @@
         assert len(w) == 1, [str(warning) for warning in w]
         assert r in str(w[0])
 
+    def test_invalid_fd(self):
+        import _socket
+        raises(ValueError, _socket.socket, fileno=-1)
+
+    def test_socket_non_inheritable(self):
+        import _socket, posix
+        s1 = _socket.socket()
+        assert posix.get_inheritable(s1.fileno()) is False
+        s1.close()
+
+    def test_socketpair_non_inheritable(self):
+        import _socket, posix
+        if not hasattr(_socket, 'socketpair'):
+            skip("no socketpair")
+        s1, s2 = _socket.socketpair()
+        assert posix.get_inheritable(s1.fileno()) is False
+        assert posix.get_inheritable(s2.fileno()) is False
+        s1.close()
+        s2.close()
+
 
 class AppTestNetlink:
     def setup_class(cls):
@@ -826,6 +854,16 @@
         assert cli.family == socket.AF_INET
 
 
+    def test_accept_non_inheritable(self):
+        import _socket, posix
+        cli = _socket.socket()
+        cli.connect(self.serv.getsockname())
+        fileno, addr = self.serv._accept()
+        assert posix.get_inheritable(fileno) is False
+        posix.close(fileno)
+        cli.close()
+
+
 class AppTestErrno:
     spaceconfig = {'usemodules': ['_socket']}
 
diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20140917)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'MAXGROUPS':      'space.wrap(interp_sre.MAXGROUPS)',
         'compile':        'interp_sre.W_SRE_Pattern',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -356,9 +356,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -458,7 +464,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -610,7 +616,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -154,6 +154,7 @@
 
     def test_readValues(self):
         from winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -167,7 +168,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,7 +119,7 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 Py_CLEANUP_SUPPORTED
@@ -975,12 +975,14 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
     global py_fatalerror
     py_fatalerror = rffi.llexternal('%s_FatalError' % prefix,
                                     [CONST_STRING], lltype.Void,
                                     compilation_info=eci)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
     add_fork_hook('child', reinit_tls)
 
 def init_function(func):
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.4.1-alpha0"
+#define PYPY_VERSION_NUM  0x05040100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -1,3 +1,6 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
 #include <Python.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -10,7 +13,7 @@
 /* Structure defines a 1-dimensional strided array */
 typedef struct{
     int* arr;
-    long length;
+    Py_ssize_t length;
 } MyArray;
 
 /* initialize the array with integers 0...length */
@@ -61,13 +64,13 @@
 static int
 PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
 {
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
     // init may have already been called
     if (self->arr.arr != NULL) {
         deallocate_MyArray(&self->arr);
     }
 
-    int length = 0;
-    static char *kwlist[] = {"length", NULL};
     if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
         return -1;
 
@@ -103,16 +106,19 @@
 static int
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
 
-  PyMyArray* self = (PyMyArray*)obj;
   view->obj = (PyObject*)self;
   view->buf = (void*)self->arr.arr;
   view->len = self->arr.length * sizeof(int);
@@ -192,7 +198,7 @@
     "buffer_test",
     "Module Doc",
     -1,
-    buffer_functions;
+    buffer_functions,
     NULL,
     NULL,
     NULL,
@@ -218,7 +224,6 @@
 #ifdef __GNUC__
 extern __attribute__((visibility("default")))
 #else
-extern __declspec(dllexport)
 #endif
 
 PyMODINIT_FUNC
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -77,4 +77,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,13 +1,8 @@
-import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -18,7 +18,8 @@
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER)
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -386,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -721,8 +724,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -32,7 +32,7 @@
 
         f = open(self.tmp + "b", "w+")
 
-        fcntl.fcntl(f, 1, 0)
+        original = fcntl.fcntl(f, 1, 0)
         fcntl.fcntl(f, 1)
         fcntl.fcntl(F(int(f.fileno())), 1)
         raises(TypeError, fcntl.fcntl, "foo")
@@ -46,9 +46,16 @@
         raises(ValueError, fcntl.fcntl, -1, 1, 0)
         raises(ValueError, fcntl.fcntl, F(-1), 1, 0)
         raises(ValueError, fcntl.fcntl, F(int(-1)), 1, 0)
-        assert fcntl.fcntl(f, 1, 0) == 0
+        assert fcntl.fcntl(f, 1, 0) == original
         assert fcntl.fcntl(f, 2, "foo") == b"foo"
-        assert fcntl.fcntl(f, 2, memoryview(b"foo")) == b"foo"
+        assert fcntl.fcntl(f, 2, b"foo") == b"foo"
+
+        # This is supposed to work I think, but CPython 3.5 refuses it
+        # for reasons I don't understand:
+        #     >>> _testcapi.getargs_s_hash(memoryview(b"foo"))
+        #     TypeError: must be read-only bytes-like object, not memoryview
+        #
+        # assert fcntl.fcntl(f, 2, memoryview(b"foo")) == b"foo"
 
         try:
             os.O_LARGEFILE
@@ -202,7 +209,7 @@
         raises(TypeError, fcntl.ioctl, "foo")
         raises(TypeError, fcntl.ioctl, 0, "foo")
         #raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, float(0))
-        raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, 1, "foo")
+        raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, 1, "foo", "bar")
 
         child_pid, mfd = pty.fork()
         if child_pid == 0:
@@ -229,13 +236,13 @@
             assert res == 0
             assert buf.tostring() == expected
 
-            exc = raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, memoryview(b'abc'), False)
-            assert str(exc.value) == "ioctl requires a file or file descriptor, an integer and optionally an integer or buffer argument"
+            raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, (), False)
 
             res = fcntl.ioctl(mfd, TIOCGPGRP, buf, False)
             assert res == expected
 
-            raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, "\x00\x00", True)
+            # xxx this fails on CPython 3.5, that's a minor bug
+            #raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, "\x00\x00", True)
 
             res = fcntl.ioctl(mfd, TIOCGPGRP, "\x00\x00\x00\x00")
             assert res == expected
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -24,7 +24,11 @@
 PREFIX = 'pypy3-'
 DEFAULT_SOABI = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])
 
-PYC_TAG = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])
+PYC_TAG = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])   # 'pypy3-XY'
+
+# see also pypy_incremental_magic in interpreter/pycode.py for the magic
+# version number stored inside pyc files.
+
 
 @specialize.memo()
 def get_so_extension(space):
@@ -228,7 +232,7 @@
 #     CPython + 7 = default_magic  -- used by PyPy (incompatible!)
 #
 from pypy.interpreter.pycode import default_magic
-MARSHAL_VERSION_FOR_PYC = 2
+MARSHAL_VERSION_FOR_PYC = 4
 
 def get_pyc_magic(space):
     return default_magic
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1188,14 +1188,8 @@
             sys.path_hooks.pop()
 
     def test_meta_path_import_error_1(self):
-        # as far as I can tell, the problem is that in CPython, if you
-        # use an import hook that doesn't update sys.modules, then the
-        # import succeeds; but at the same time, you can have the same
-        # result without an import hook (see test_del_from_sys_modules)
-        # and then the import fails.  This looks like even more mess
-        # to replicate, so we ignore it until someone really hits this
-        # case...
-        skip("looks like an inconsistency in CPython")
+        # check that we get a KeyError somewhere inside
+        # <frozen importlib._bootstrap>, like CPython 3.5
 
         class ImportHook(object):
             def find_module(self, fullname, path=None):
@@ -1205,12 +1199,12 @@
             def load_module(self, fullname):
                 assert fullname == 'meta_path_pseudo_module'
                 # we "forget" to update sys.modules
-                return new.module('meta_path_pseudo_module')
+                return types.ModuleType('meta_path_pseudo_module')
 
-        import sys, new
+        import sys, types
         sys.meta_path.append(ImportHook())
         try:
-            import meta_path_pseudo_module
+            raises(KeyError, "import meta_path_pseudo_module")
         finally:
             sys.meta_path.pop()
 
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -4,24 +4,30 @@
 from rpython.rlib import rstackovf
 from pypy.objspace.std.marshal_impl import marshal, get_unmarshallers
 
+#
+# Write Python objects to files and read them back.  This is primarily
+# intended for writing and reading compiled Python code, even though
+# dicts, lists, sets and frozensets, not commonly seen in code
+# objects, are supported.  Version 3 of this protocol properly
+# supports circular links and sharing.  The previous version is called
+# "2", like in Python 2.7, although it is not always compatible
+# between CPython 2.7 and CPython 3.4.  Version 4 adds small
+# optimizations in compactness.
+#
+# XXX: before py3k, there was logic to do efficiently dump()/load() on
+# a file object.  The corresponding logic is gone from CPython 3.x, so
+# I don't feel bad about killing it here too.
+#
 
-Py_MARSHAL_VERSION = 2
+Py_MARSHAL_VERSION = 4
+
 
 @unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION))
 def dump(space, w_data, w_f, w_version):
     """Write the 'data' object into the open file 'f'."""
-    # XXX: before py3k, we special-cased W_File to use a more performant
-    # FileWriter class. Should we do the same for py3k? Look also at
-    # DirectStreamWriter
-    writer = FileWriter(space, w_f)
-    try:
-        # note: bound methods are currently not supported,
-        # so we have to pass the instance in, instead.
-        ##m = Marshaller(space, writer.write, space.int_w(w_version))
-        m = Marshaller(space, writer, space.int_w(w_version))
-        m.dump_w_obj(w_data)

From pypy.commits at gmail.com  Mon Aug 29 07:48:38 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 04:48:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: missing test code in last merge
Message-ID: <57c42116.c3f0c20a.cb08.2cb6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86685:981849b5f33c
Date: 2016-08-29 13:08 +0200
http://bitbucket.org/pypy/pypy/changeset/981849b5f33c/

Log:	missing test code in last merge

diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -177,3 +177,178 @@
 
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
+
+class MockBuffer(Buffer):
+    def __init__(self, space, w_arr, w_dim, w_fmt, \
+                 w_itemsize, w_strides, w_shape):
+        self.space = space
+        self.w_arr = w_arr
+        self.arr = []
+        self.ndim = space.int_w(w_dim)
+        self.format = space.str_w(w_fmt)
+        self.itemsize = space.int_w(w_itemsize)
+        self.strides = []
+        for w_i in w_strides.getitems_unroll():
+            self.strides.append(space.int_w(w_i))
+        self.shape = []
+        for w_i in w_shape.getitems_unroll():
+            self.shape.append(space.int_w(w_i))
+        self.readonly = True
+        self.shape.append(space.len_w(w_arr))
+        self.data = []
+        itemsize = 1
+        worklist = [(1,w_arr)]
+        while worklist:
+            dim, w_work = worklist.pop()
+            if space.isinstance_w(w_work, space.w_list):
+                for j, w_obj in enumerate(w_work.getitems_unroll()):
+                    worklist.insert(0, (dim+1, w_obj))
+                continue
+            byte = struct.pack(self.format, space.int_w(w_work))
+            for c in byte:
+                self.data.append(c)
+        self.data = ''.join(self.data)
+
+    def getslice(self, start, stop, step, size):
+        items = []
+        if size == 0:
+            return ''
+        return ''.join([self.getitem(i) for i in range(start,stop,step)])
+        #bytecount = (stop - start)
+        ## data is stores as list of ints, thus this gets around the
+        ## issue that one cannot advance in bytes
+        #count = bytecount // size
+        #start = start // size
+        #for i in range(start, start+count, step):
+        #    items.append(self.getitem(i))
+        #return ''.join(items)
+
+
+    def getformat(self):
+        return self.format
+
+    def getitem(self, index):
+        return self.data[index:index+1]
+
+    def getlength(self):
+        return len(self.data)
+
+    def getitemsize(self):
+        return self.itemsize
+
+    def getndim(self):
+        return self.ndim
+
+    def getstrides(self):
+        return self.strides
+
+    def getshape(self):
+        return self.shape
+
+    def is_contiguous(self, format):
+        return format == 'C'
+
+class W_MockArray(W_Root):
+    def __init__(self, w_list, w_dim, w_fmt, w_size, w_strides, w_shape):
+        self.w_list = w_list
+        self.w_dim = w_dim
+        self.w_fmt = w_fmt
+        self.w_size = w_size
+        self.w_strides = w_strides
+        self.w_shape = w_shape
+
+    @staticmethod
+    def descr_new(space, w_type, w_list, w_dim, w_fmt, \
+                         w_size, w_strides, w_shape):
+        return W_MockArray(w_list, w_dim, w_fmt, w_size, w_strides, w_shape)
+
+    def buffer_w(self, space, flags):
+        return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
+                          self.w_size, self.w_strides, self.w_shape)
+
+    def buffer_w_ex(self, space, flags):
+        return self.buffer_w(space, flags), space.str_w(self.w_fmt), space.int_w(self.w_size)
+
+W_MockArray.typedef = TypeDef("MockArray",
+    __new__ = interp2app(W_MockArray.descr_new),
+)
+
+from pypy.objspace.std.transparent import register_proxyable
+from pypy.conftest import option
+
+class AppTestMemoryViewMockBuffer(object):
+    spaceconfig = dict(usemodules=[])
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("Impossible to run on appdirect")
+        cls.w_MockArray = cls.space.gettypefor(W_MockArray)
+
+    def test_tuple_indexing(self):
+        content = self.MockArray([[0,1,2,3], [4,5,6,7], [8,9,10,11]],
+                                 dim=2, fmt='B', size=1,
+                                 strides=[4,1], shape=[3,4])
+        view = memoryview(content)
+        assert view[0,0] == 0
+        assert view[2,0] == 8
+        assert view[2,3] == 11
+        assert view[-1,-1] == 11
+        assert view[-3,-4] == 0
+
+        try:
+            view.__getitem__((2**63-1,0))
+            assert False, "must not succeed"
+        except IndexError: pass
+
+        try:
+            view.__getitem__((0, 0, 0))
+            assert False, "must not succeed"
+        except TypeError: pass
+
+    def test_tuple_indexing_int(self):
+        content = self.MockArray([ [[1],[2],[3]], [[4],[5],[6]] ],
+                                 dim=3, fmt='i', size=4,
+                                 strides=[12,4,4], shape=[2,3,1])
+        view = memoryview(content)
+        assert view[0,0,0] == 1
+        assert view[-1,2,0] == 6
+
+    def test_cast_non_byte(self):
+        empty = self.MockArray([], dim=1, fmt='i', size=4, strides=[1], shape=[1])
+        view = memoryview(empty)
+        try:
+            view.cast('l')
+            assert False, "i -> l not possible. buffer must be byte format"
+        except TypeError:
+            pass
+
+    def test_cast_empty(self):
+        empty = self.MockArray([], dim=1, fmt='b', size=1, strides=[1], shape=[1])
+        view = memoryview(empty)
+        cview = view.cast('i')
+        assert cview.tobytes() == b''
+        assert cview.tolist() == []
+        assert view.format == 'b'
+        assert cview.format == 'i'
+        #
+        #assert cview.cast('i').cast('b').cast('i').tolist() == []
+        #
+        assert cview.format == 'i'
+        try:
+            cview.cast('i')
+            assert False, "cast must fail"
+        except TypeError:
+            pass
+
+    def test_cast_with_shape(self):
+        empty = self.MockArray([1,0,2,0,3,0],
+                    dim=1, fmt='h', size=2,
+                    strides=[8], shape=[6])
+        view = memoryview(empty)
+        byteview = view.cast('b')
+        #assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]
+        i32view = byteview.cast('i', shape=[1,3])
+        assert i32view.format == 'i'
+        assert i32view.itemsize == 4
+        assert i32view.tolist() == [[1,2,3]]
+        i32view = byteview.cast('i', shape=(1,3))
+        assert i32view.tolist() == [[1,2,3]]

From pypy.commits at gmail.com  Mon Aug 29 07:48:40 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 04:48:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: new test,
 some more fixes after the merge
Message-ID: <57c42118.ca11c30a.3a3c4.2dec@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86686:1d9d5dc5f9bb
Date: 2016-08-29 13:48 +0200
http://bitbucket.org/pypy/pypy/changeset/1d9d5dc5f9bb/

Log:	new test, some more fixes after the merge

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -11,6 +11,7 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
 from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
+from pypy.objspace.std.bytesobject import getbytevalue
 from rpython.rlib.unroll import unrolling_iterable
 
 MEMORYVIEW_MAX_DIM = 64
@@ -149,12 +150,11 @@
         if dim >= self.getndim():
             bytecount = (stride * dimshape)
             count = bytecount // itemsize
-            return self._tolist(space, SubBuffer(buf, start, bytecount), count, fmt)
+            return self._tolist(space, buf, count, fmt)
         items = [None] * dimshape
 
         for i in range(dimshape):
-            bytecount = stride
-            item = self._tolist_rec(space, SubBuffer(buf, start, bytecount), start, idim+1, fmt)
+            item = self._tolist_rec(space, SubBuffer(buf, start, stride), start, idim+1, fmt)
             items[i] = item
             start += stride
 
@@ -223,7 +223,7 @@
 
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
-	itemsize = self.getitemsize()
+        itemsize = self.getitemsize()
         if itemsize > 1:
             start *= itemsize
             size *= itemsize
@@ -239,13 +239,13 @@
                 return space.newint(ord(ch))
             else:
                 # TODO: this probably isn't very fast
-                buf = SubBuffer(self.buf, start * itemsize, itemsize)
+                buf = SubBuffer(self.buf, start, itemsize)
                 fmtiter = UnpackFormatIterator(space, buf)
                 fmtiter.interpret(self.format)
                 return fmtiter.result_w[0]
         elif step == 1:
             buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf, self.getformat(), self.itemsize)
+            return W_MemoryView(buf, self.getformat(), itemsize)
         else:
             # XXX needs to return a W_MemoryView with a NonContiguousSubBuffer
             # maybe?  Need to check the cpyext requirements for that
@@ -408,7 +408,6 @@
         return False
 
     def descr_cast(self, space, w_format, w_shape=None):
-        # XXX fixme. does not do anything near cpython (see memoryobjet.c memory_cast)
         self._check_released(space)
 
         if not space.isinstance_w(w_format, space.w_unicode):
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef
 from rpython.rlib.buffer import Buffer
+from pypy.conftest import option
 
 class AppTestMemoryView:
     spaceconfig = dict(usemodules=['array'])
@@ -273,9 +274,6 @@
     __new__ = interp2app(W_MockArray.descr_new),
 )
 
-from pypy.objspace.std.transparent import register_proxyable
-from pypy.conftest import option
-
 class AppTestMemoryViewMockBuffer(object):
     spaceconfig = dict(usemodules=[])
     def setup_class(cls):
@@ -352,3 +350,16 @@
         assert i32view.tolist() == [[1,2,3]]
         i32view = byteview.cast('i', shape=(1,3))
         assert i32view.tolist() == [[1,2,3]]
+
+    def test_cast_bytes(self):
+        bytes = b"\x02\x00\x03\x00\x04\x00" \
+                b"\x05\x00\x06\x00\x07\x00"
+        view = memoryview(bytes)
+        v = view.cast('h', shape=(3,2))
+        assert v.tolist() == [[2,3],[4,5],[6,7]]
+        try:
+            v = view.cast('h', shape=(3,3))
+            assert False, "shape is too big for bytes"
+        except TypeError:
+            pass
+

From pypy.commits at gmail.com  Mon Aug 29 07:52:58 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 Aug 2016 04:52:58 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Moved tag release-pypy2.7-v5.4.0 to
 changeset 77392ad26350 (from changeset 68bb3510d821)
Message-ID: <57c4221a.eeb8c20a.90862.2c74@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86688:e709f9b755b4
Date: 2016-08-29 21:46 +1000
http://bitbucket.org/pypy/pypy/changeset/e709f9b755b4/

Log:	Moved tag release-pypy2.7-v5.4.0 to changeset 77392ad26350 (from
	changeset 68bb3510d821)

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,5 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0

From pypy.commits at gmail.com  Mon Aug 29 07:53:00 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 Aug 2016 04:53:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <57c4221c.c2a5c20a.46c00.2682@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86689:7ccea82f1b25
Date: 2016-08-29 21:50 +1000
http://bitbucket.org/pypy/pypy/changeset/7ccea82f1b25/

Log:	merge heads

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -28,3 +28,5 @@
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0

From pypy.commits at gmail.com  Mon Aug 29 07:53:02 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 Aug 2016 04:53:02 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: merge heads
Message-ID: <57c4221e.0117c20a.c7a53.3ed7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86690:c85b4cec18fd
Date: 2016-08-29 21:51 +1000
http://bitbucket.org/pypy/pypy/changeset/c85b4cec18fd/

Log:	merge heads


From pypy.commits at gmail.com  Mon Aug 29 07:52:57 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 Aug 2016 04:52:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: Moved tag release-pypy2.7-v5.4.0 to
 changeset 77392ad26350 (from changeset 68bb3510d821)
Message-ID: <57c42219.4219c20a.987dd.38d1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86687:00eb13b6882d
Date: 2016-08-29 21:46 +1000
http://bitbucket.org/pypy/pypy/changeset/00eb13b6882d/

Log:	Moved tag release-pypy2.7-v5.4.0 to changeset 77392ad26350 (from
	changeset 68bb3510d821)

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -28,3 +28,5 @@
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0

From pypy.commits at gmail.com  Mon Aug 29 08:07:06 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 05:07:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: readded missing hex method to
 bytes. seemed it got lost during a merge
Message-ID: <57c4256a.0117c20a.c7a53.44f0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86691:715ec3109ab7
Date: 2016-08-29 14:06 +0200
http://bitbucket.org/pypy/pypy/changeset/715ec3109ab7/

Log:	readded missing hex method to bytes. seemed it got lost during a
	merge

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -659,6 +659,10 @@
     def descr_upper(self, space):
         return W_BytesObject(self._value.upper())
 
+    def descr_hex(self, space):
+        from pypy.objspace.std.bytearrayobject import _array_to_hexstring
+        return _array_to_hexstring(space, StringBuffer(self._value))
+
 
 def _create_list_from_bytes(value):
     # need this helper function to allow the jit to look inside and inline
@@ -838,6 +842,7 @@
 
     fromhex = interp2app(W_BytesObject.descr_fromhex, as_classmethod=True),
     maketrans = interp2app(W_BytesObject.descr_maketrans, as_classmethod=True),
+    hex = interp2app(W_BytesObject.descr_hex),
 )
 W_BytesObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -210,21 +210,6 @@
                 self.data.append(c)
         self.data = ''.join(self.data)
 
-    def getslice(self, start, stop, step, size):
-        items = []
-        if size == 0:
-            return ''
-        return ''.join([self.getitem(i) for i in range(start,stop,step)])
-        #bytecount = (stop - start)
-        ## data is stores as list of ints, thus this gets around the
-        ## issue that one cannot advance in bytes
-        #count = bytecount // size
-        #start = start // size
-        #for i in range(start, start+count, step):
-        #    items.append(self.getitem(i))
-        #return ''.join(items)
-
-
     def getformat(self):
         return self.format
 

From pypy.commits at gmail.com  Mon Aug 29 08:51:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 05:51:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: Support for re.fullpatch(),
 for Python >= 3.4
Message-ID: <57c42fbd.c186c20a.299e7.3d2b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86692:6d29597bf1ac
Date: 2016-08-29 14:36 +0200
http://bitbucket.org/pypy/pypy/changeset/6d29597bf1ac/

Log:	Support for re.fullpatch(), for Python >= 3.4

diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -52,6 +52,7 @@
 SRE_INFO_CHARSET = 4
 SRE_FLAG_LOCALE = 4 # honour system locale
 SRE_FLAG_UNICODE = 32 # use unicode locale
+SRE_FLAG_FULLMATCH = 0x4000   # PyPy extension, for CPython >= 3.4
 
 
 def getlower(char_ord, flags):
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -526,9 +526,16 @@
         if op == OPCODE_FAILURE:
             return
 
-        if (op == OPCODE_SUCCESS or
-            op == OPCODE_MAX_UNTIL or
-            op == OPCODE_MIN_UNTIL):
+        elif op == OPCODE_SUCCESS:
+            if ctx.flags & rsre_char.SRE_FLAG_FULLMATCH:
+                if ptr != ctx.end:
+                    return     # not a full match
+            ctx.match_end = ptr
+            ctx.match_marks = marks
+            return MATCHED_OK
+
+        elif (op == OPCODE_MAX_UNTIL or
+              op == OPCODE_MIN_UNTIL):
             ctx.match_end = ptr
             ctx.match_marks = marks
             return MATCHED_OK
@@ -1007,6 +1014,10 @@
     else:
         return None
 
+def fullmatch(pattern, string, start=0, end=sys.maxint, flags=0):
+    return match(pattern, string, start, end,
+                 flags | rsre_char.SRE_FLAG_FULLMATCH)
+
 def search(pattern, string, start=0, end=sys.maxint, flags=0):
     start, end = _adjust(start, end, len(string))
     ctx = StrMatchContext(pattern, string, start, end, flags)
diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py
--- a/rpython/rlib/rsre/test/test_match.py
+++ b/rpython/rlib/rsre/test/test_match.py
@@ -272,3 +272,24 @@
         r = get_code("\\{\\{((?:.*?)+)\\}\\}")
         match = rsre_core.match(r, "{{a}}{{b}}")
         assert match.group(1) == "a"
+
+    def test_fullmatch_1(self):
+        r = get_code(r"ab*c")
+        assert not rsre_core.fullmatch(r, "abbbcdef")
+        assert rsre_core.fullmatch(r, "abbbc")
+
+    def test_fullmatch_2(self):
+        r = get_code(r"a(b*?)")
+        match = rsre_core.fullmatch(r, "abbb")
+        assert match.group(1) == "bbb"
+        assert not rsre_core.fullmatch(r, "abbbc")
+
+    def test_fullmatch_3(self):
+        r = get_code(r"a((bp)*?)c")
+        match = rsre_core.fullmatch(r, "abpbpbpc")
+        assert match.group(1) == "bpbpbp"
+
+    def test_fullmatch_4(self):
+        r = get_code(r"a((bp)*)c")
+        match = rsre_core.fullmatch(r, "abpbpbpc")
+        assert match.group(1) == "bpbpbp"

From pypy.commits at gmail.com  Mon Aug 29 08:51:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 05:51:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c42fbf.531d1c0a.94f66.7b9c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86693:478944df2b79
Date: 2016-08-29 14:36 +0200
http://bitbucket.org/pypy/pypy/changeset/478944df2b79/

Log:	hg merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -28,3 +28,5 @@
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -52,6 +52,7 @@
 SRE_INFO_CHARSET = 4
 SRE_FLAG_LOCALE = 4 # honour system locale
 SRE_FLAG_UNICODE = 32 # use unicode locale
+SRE_FLAG_FULLMATCH = 0x4000   # PyPy extension, for CPython >= 3.4
 
 
 def getlower(char_ord, flags):
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -526,9 +526,16 @@
         if op == OPCODE_FAILURE:
             return
 
-        if (op == OPCODE_SUCCESS or
-            op == OPCODE_MAX_UNTIL or
-            op == OPCODE_MIN_UNTIL):
+        elif op == OPCODE_SUCCESS:
+            if ctx.flags & rsre_char.SRE_FLAG_FULLMATCH:
+                if ptr != ctx.end:
+                    return     # not a full match
+            ctx.match_end = ptr
+            ctx.match_marks = marks
+            return MATCHED_OK
+
+        elif (op == OPCODE_MAX_UNTIL or
+              op == OPCODE_MIN_UNTIL):
             ctx.match_end = ptr
             ctx.match_marks = marks
             return MATCHED_OK
@@ -1007,6 +1014,10 @@
     else:
         return None
 
+def fullmatch(pattern, string, start=0, end=sys.maxint, flags=0):
+    return match(pattern, string, start, end,
+                 flags | rsre_char.SRE_FLAG_FULLMATCH)
+
 def search(pattern, string, start=0, end=sys.maxint, flags=0):
     start, end = _adjust(start, end, len(string))
     ctx = StrMatchContext(pattern, string, start, end, flags)
diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py
--- a/rpython/rlib/rsre/test/test_match.py
+++ b/rpython/rlib/rsre/test/test_match.py
@@ -272,3 +272,24 @@
         r = get_code("\\{\\{((?:.*?)+)\\}\\}")
         match = rsre_core.match(r, "{{a}}{{b}}")
         assert match.group(1) == "a"
+
+    def test_fullmatch_1(self):
+        r = get_code(r"ab*c")
+        assert not rsre_core.fullmatch(r, "abbbcdef")
+        assert rsre_core.fullmatch(r, "abbbc")
+
+    def test_fullmatch_2(self):
+        r = get_code(r"a(b*?)")
+        match = rsre_core.fullmatch(r, "abbb")
+        assert match.group(1) == "bbb"
+        assert not rsre_core.fullmatch(r, "abbbc")
+
+    def test_fullmatch_3(self):
+        r = get_code(r"a((bp)*?)c")
+        match = rsre_core.fullmatch(r, "abpbpbpc")
+        assert match.group(1) == "bpbpbp"
+
+    def test_fullmatch_4(self):
+        r = get_code(r"a((bp)*)c")
+        match = rsre_core.fullmatch(r, "abpbpbpc")
+        assert match.group(1) == "bpbpbp"

From pypy.commits at gmail.com  Mon Aug 29 08:51:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 05:51:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c42fc1.531d1c0a.94f66.7ba3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86694:c00dea582ece
Date: 2016-08-29 14:36 +0200
http://bitbucket.org/pypy/pypy/changeset/c00dea582ece/

Log:	hg merge py3k

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -28,3 +28,5 @@
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1159,23 +1159,18 @@
     # 'flags' might be ignored.  Check the result.
     if _WIN32:
         # 'flags' ignored
-        pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        try:
-            ok = CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0)
-            hread = rffi.cast(rffi.INTPTR_T, pread[0])
-            hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
-        finally:
-            lltype.free(pwrite, flavor='raw')
-            lltype.free(pread, flavor='raw')
-        if ok:
-            fdread = c_open_osfhandle(hread, 0)
-            fdwrite = c_open_osfhandle(hwrite, 1)
-            if not (fdread == -1 or fdwrite == -1):
-                return (fdread, fdwrite)
-            rwin32.CloseHandle(pread)
-            rwin32.CloseHandle(pwrite)
+        ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
+        walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
+        with ralloc as pread, walloc as pwrite:
+            if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
+                fdread = c_open_osfhandle(
+                    rffi.cast(rffi.INTPTR_T, pread[0]), 0)
+                fdwrite = c_open_osfhandle(
+                    rffi.cast(rffi.INTPTR_T, pwrite[0]), 1)
+                if not (fdread == -1 or fdwrite == -1):
+                    return (fdread, fdwrite)
+                rwin32.CloseHandle(pread)
+                rwin32.CloseHandle(pwrite)
         raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -52,6 +52,7 @@
 SRE_INFO_CHARSET = 4
 SRE_FLAG_LOCALE = 4 # honour system locale
 SRE_FLAG_UNICODE = 32 # use unicode locale
+SRE_FLAG_FULLMATCH = 0x4000   # PyPy extension, for CPython >= 3.4
 
 
 def getlower(char_ord, flags):
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -526,9 +526,16 @@
         if op == OPCODE_FAILURE:
             return
 
-        if (op == OPCODE_SUCCESS or
-            op == OPCODE_MAX_UNTIL or
-            op == OPCODE_MIN_UNTIL):
+        elif op == OPCODE_SUCCESS:
+            if ctx.flags & rsre_char.SRE_FLAG_FULLMATCH:
+                if ptr != ctx.end:
+                    return     # not a full match
+            ctx.match_end = ptr
+            ctx.match_marks = marks
+            return MATCHED_OK
+
+        elif (op == OPCODE_MAX_UNTIL or
+              op == OPCODE_MIN_UNTIL):
             ctx.match_end = ptr
             ctx.match_marks = marks
             return MATCHED_OK
@@ -1007,6 +1014,10 @@
     else:
         return None
 
+def fullmatch(pattern, string, start=0, end=sys.maxint, flags=0):
+    return match(pattern, string, start, end,
+                 flags | rsre_char.SRE_FLAG_FULLMATCH)
+
 def search(pattern, string, start=0, end=sys.maxint, flags=0):
     start, end = _adjust(start, end, len(string))
     ctx = StrMatchContext(pattern, string, start, end, flags)
diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py
--- a/rpython/rlib/rsre/test/test_match.py
+++ b/rpython/rlib/rsre/test/test_match.py
@@ -272,3 +272,24 @@
         r = get_code("\\{\\{((?:.*?)+)\\}\\}")
         match = rsre_core.match(r, "{{a}}{{b}}")
         assert match.group(1) == "a"
+
+    def test_fullmatch_1(self):
+        r = get_code(r"ab*c")
+        assert not rsre_core.fullmatch(r, "abbbcdef")
+        assert rsre_core.fullmatch(r, "abbbc")
+
+    def test_fullmatch_2(self):
+        r = get_code(r"a(b*?)")
+        match = rsre_core.fullmatch(r, "abbb")
+        assert match.group(1) == "bbb"
+        assert not rsre_core.fullmatch(r, "abbbc")
+
+    def test_fullmatch_3(self):
+        r = get_code(r"a((bp)*?)c")
+        match = rsre_core.fullmatch(r, "abpbpbpc")
+        assert match.group(1) == "bpbpbp"
+
+    def test_fullmatch_4(self):
+        r = get_code(r"a((bp)*)c")
+        match = rsre_core.fullmatch(r, "abpbpbpc")
+        assert match.group(1) == "bpbpbp"

From pypy.commits at gmail.com  Mon Aug 29 08:51:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 05:51:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: re.fullmatch()
Message-ID: <57c42fc3.05d71c0a.351b2.1f7b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86695:2318b859f5f6
Date: 2016-08-29 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/2318b859f5f6/

Log:	re.fullmatch()

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,8 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower
+from rpython.rlib.rsre.rsre_char import set_unicode_db, SRE_FLAG_FULLMATCH
 
 
 @unwrap_spec(char_ord=int, flags=int)
@@ -99,7 +100,7 @@
         space = self.space
         raise oefmt(space.w_TypeError, "cannot copy this pattern object")
 
-    def make_ctx(self, w_string, pos=0, endpos=sys.maxint):
+    def make_ctx(self, w_string, pos=0, endpos=sys.maxint, flags=0):
         """Make a StrMatchContext, BufMatchContext or a UnicodeMatchContext for
         searching in the given w_string object."""
         space = self.space
@@ -107,6 +108,7 @@
             pos = 0
         if endpos < pos:
             endpos = pos
+        flags = self.flags | flags
         if space.isinstance_w(w_string, space.w_unicode):
             unicodestr = space.unicode_w(w_string)
             if not (space.is_none(self.w_pattern) or
@@ -119,7 +121,7 @@
             if endpos > len(unicodestr):
                 endpos = len(unicodestr)
             return rsre_core.UnicodeMatchContext(self.code, unicodestr,
-                                                 pos, endpos, self.flags)
+                                                 pos, endpos, flags)
         elif space.isinstance_w(w_string, space.w_str):
             if (not space.is_none(self.w_pattern) and
                 space.isinstance_w(self.w_pattern, space.w_unicode)):
@@ -132,7 +134,7 @@
             if endpos > len(str):
                 endpos = len(str)
             return rsre_core.StrMatchContext(self.code, str,
-                                             pos, endpos, self.flags)
+                                             pos, endpos, flags)
         else:
             buf = space.readbuf_w(w_string)
             if (not space.is_none(self.w_pattern) and
@@ -147,7 +149,7 @@
             if endpos > size:
                 endpos = size
             return rsre_core.BufMatchContext(self.code, buf,
-                                             pos, endpos, self.flags)
+                                             pos, endpos, flags)
 
     def getmatch(self, ctx, found):
         if found:
@@ -161,6 +163,11 @@
         return self.getmatch(ctx, matchcontext(self.space, ctx))
 
     @unwrap_spec(pos=int, endpos=int)
+    def fullmatch_w(self, w_string, pos=0, endpos=sys.maxint):
+        ctx = self.make_ctx(w_string, pos, endpos, SRE_FLAG_FULLMATCH)
+        return self.getmatch(ctx, matchcontext(self.space, ctx))
+
+    @unwrap_spec(pos=int, endpos=int)
     def search_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
         return self.getmatch(ctx, searchcontext(self.space, ctx))
@@ -419,6 +426,7 @@
     findall      = interp2app(W_SRE_Pattern.findall_w),
     finditer     = interp2app(W_SRE_Pattern.finditer_w),
     match        = interp2app(W_SRE_Pattern.match_w),
+    fullmatch    = interp2app(W_SRE_Pattern.fullmatch_w),
     scanner      = interp2app(W_SRE_Pattern.finditer_w),    # reuse finditer()
     search       = interp2app(W_SRE_Pattern.search_w),
     split        = interp2app(W_SRE_Pattern.split_w),
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -116,6 +116,13 @@
         import _sre
         raises(TypeError, _sre.compile, {}, 0, [])
 
+    def test_fullmatch(self):
+        import re
+        assert re.compile(r"ab*c").fullmatch("abbcdef") is None
+        assert re.compile(r"ab*c").fullmatch("abbc") is not None
+        assert re.fullmatch(r"ab*c", "abbbcdef") is None
+        assert re.fullmatch(r"ab*c", "abbbc") is not None
+
 
 class AppTestSreMatch:
     spaceconfig = dict(usemodules=('array', ))

From pypy.commits at gmail.com  Mon Aug 29 10:14:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 07:14:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: test and fix
Message-ID: <57c44331.8aacc20a.f3495.64e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86696:0262db87059d
Date: 2016-08-29 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/0262db87059d/

Log:	test and fix

diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -52,7 +52,6 @@
 SRE_INFO_CHARSET = 4
 SRE_FLAG_LOCALE = 4 # honour system locale
 SRE_FLAG_UNICODE = 32 # use unicode locale
-SRE_FLAG_FULLMATCH = 0x4000   # PyPy extension, for CPython >= 3.4
 
 
 def getlower(char_ord, flags):
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -89,6 +89,7 @@
     match_end = 0
     match_marks = None
     match_marks_flat = None
+    fullmatch_only = False
 
     def __init__(self, pattern, match_start, end, flags):
         # 'match_start' and 'end' must be known to be non-negative
@@ -527,7 +528,7 @@
             return
 
         elif op == OPCODE_SUCCESS:
-            if ctx.flags & rsre_char.SRE_FLAG_FULLMATCH:
+            if ctx.fullmatch_only:
                 if ptr != ctx.end:
                     return     # not a full match
             ctx.match_end = ptr
@@ -558,7 +559,11 @@
             # assert subpattern
             # <ASSERT> <0=skip> <1=back> <pattern>
             ptr1 = ptr - ctx.pat(ppos+1)
-            if ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None:
+            saved = ctx.fullmatch_only
+            ctx.fullmatch_only = False
+            stop = ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None
+            ctx.fullmatch_only = saved
+            if stop:
                 return
             marks = ctx.match_marks
             ppos += ctx.pat(ppos)
@@ -567,7 +572,12 @@
             # assert not subpattern
             # <ASSERT_NOT> <0=skip> <1=back> <pattern>
             ptr1 = ptr - ctx.pat(ppos+1)
-            if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) is not None:
+            saved = ctx.fullmatch_only
+            ctx.fullmatch_only = False
+            stop = (ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks)
+                                      is not None)
+            ctx.fullmatch_only = saved
+            if stop:
                 return
             ppos += ctx.pat(ppos)
 
@@ -1006,17 +1016,17 @@
     elif end > length: end = length
     return start, end
 
-def match(pattern, string, start=0, end=sys.maxint, flags=0):
+def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False):
     start, end = _adjust(start, end, len(string))
     ctx = StrMatchContext(pattern, string, start, end, flags)
+    ctx.fullmatch_only = fullmatch
     if match_context(ctx):
         return ctx
     else:
         return None
 
 def fullmatch(pattern, string, start=0, end=sys.maxint, flags=0):
-    return match(pattern, string, start, end,
-                 flags | rsre_char.SRE_FLAG_FULLMATCH)
+    return match(pattern, string, start, end, flags, fullmatch=True)
 
 def search(pattern, string, start=0, end=sys.maxint, flags=0):
     start, end = _adjust(start, end, len(string))
diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py
--- a/rpython/rlib/rsre/test/test_match.py
+++ b/rpython/rlib/rsre/test/test_match.py
@@ -293,3 +293,9 @@
         r = get_code(r"a((bp)*)c")
         match = rsre_core.fullmatch(r, "abpbpbpc")
         assert match.group(1) == "bpbpbp"
+
+    def test_fullmatch_assertion(self):
+        r = get_code(r"(?=a).b")
+        assert rsre_core.fullmatch(r, "ab")
+        r = get_code(r"(?!a)..")
+        assert not rsre_core.fullmatch(r, "ab")

From pypy.commits at gmail.com  Mon Aug 29 10:16:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 07:16:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c443d3.e16ec20a.9378f.ef8d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86697:57ccc10e7bfc
Date: 2016-08-29 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/57ccc10e7bfc/

Log:	hg merge default

diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -52,7 +52,6 @@
 SRE_INFO_CHARSET = 4
 SRE_FLAG_LOCALE = 4 # honour system locale
 SRE_FLAG_UNICODE = 32 # use unicode locale
-SRE_FLAG_FULLMATCH = 0x4000   # PyPy extension, for CPython >= 3.4
 
 
 def getlower(char_ord, flags):
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -89,6 +89,7 @@
     match_end = 0
     match_marks = None
     match_marks_flat = None
+    fullmatch_only = False
 
     def __init__(self, pattern, match_start, end, flags):
         # 'match_start' and 'end' must be known to be non-negative
@@ -527,7 +528,7 @@
             return
 
         elif op == OPCODE_SUCCESS:
-            if ctx.flags & rsre_char.SRE_FLAG_FULLMATCH:
+            if ctx.fullmatch_only:
                 if ptr != ctx.end:
                     return     # not a full match
             ctx.match_end = ptr
@@ -558,7 +559,11 @@
             # assert subpattern
             # <ASSERT> <0=skip> <1=back> <pattern>
             ptr1 = ptr - ctx.pat(ppos+1)
-            if ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None:
+            saved = ctx.fullmatch_only
+            ctx.fullmatch_only = False
+            stop = ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None
+            ctx.fullmatch_only = saved
+            if stop:
                 return
             marks = ctx.match_marks
             ppos += ctx.pat(ppos)
@@ -567,7 +572,12 @@
             # assert not subpattern
             # <ASSERT_NOT> <0=skip> <1=back> <pattern>
             ptr1 = ptr - ctx.pat(ppos+1)
-            if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) is not None:
+            saved = ctx.fullmatch_only
+            ctx.fullmatch_only = False
+            stop = (ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks)
+                                      is not None)
+            ctx.fullmatch_only = saved
+            if stop:
                 return
             ppos += ctx.pat(ppos)
 
@@ -1006,17 +1016,17 @@
     elif end > length: end = length
     return start, end
 
-def match(pattern, string, start=0, end=sys.maxint, flags=0):
+def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False):
     start, end = _adjust(start, end, len(string))
     ctx = StrMatchContext(pattern, string, start, end, flags)
+    ctx.fullmatch_only = fullmatch
     if match_context(ctx):
         return ctx
     else:
         return None
 
 def fullmatch(pattern, string, start=0, end=sys.maxint, flags=0):
-    return match(pattern, string, start, end,
-                 flags | rsre_char.SRE_FLAG_FULLMATCH)
+    return match(pattern, string, start, end, flags, fullmatch=True)
 
 def search(pattern, string, start=0, end=sys.maxint, flags=0):
     start, end = _adjust(start, end, len(string))
diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py
--- a/rpython/rlib/rsre/test/test_match.py
+++ b/rpython/rlib/rsre/test/test_match.py
@@ -293,3 +293,9 @@
         r = get_code(r"a((bp)*)c")
         match = rsre_core.fullmatch(r, "abpbpbpc")
         assert match.group(1) == "bpbpbp"
+
+    def test_fullmatch_assertion(self):
+        r = get_code(r"(?=a).b")
+        assert rsre_core.fullmatch(r, "ab")
+        r = get_code(r"(?!a)..")
+        assert not rsre_core.fullmatch(r, "ab")

From pypy.commits at gmail.com  Mon Aug 29 10:16:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 07:16:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: update
Message-ID: <57c443d7.a719c20a.7a270.6bf3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86699:64af2a8d31d3
Date: 2016-08-29 16:16 +0200
http://bitbucket.org/pypy/pypy/changeset/64af2a8d31d3/

Log:	update

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,8 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower
-from rpython.rlib.rsre.rsre_char import set_unicode_db, SRE_FLAG_FULLMATCH
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
@@ -164,7 +163,8 @@
 
     @unwrap_spec(pos=int, endpos=int)
     def fullmatch_w(self, w_string, pos=0, endpos=sys.maxint):
-        ctx = self.make_ctx(w_string, pos, endpos, SRE_FLAG_FULLMATCH)
+        ctx = self.make_ctx(w_string, pos, endpos)
+        ctx.fullmatch_only = True
         return self.getmatch(ctx, matchcontext(self.space, ctx))
 
     @unwrap_spec(pos=int, endpos=int)

From pypy.commits at gmail.com  Mon Aug 29 10:16:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 07:16:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c443d5.09afc20a.8df27.733d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86698:a513e5ef114e
Date: 2016-08-29 16:14 +0200
http://bitbucket.org/pypy/pypy/changeset/a513e5ef114e/

Log:	hg merge py3k

diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -52,7 +52,6 @@
 SRE_INFO_CHARSET = 4
 SRE_FLAG_LOCALE = 4 # honour system locale
 SRE_FLAG_UNICODE = 32 # use unicode locale
-SRE_FLAG_FULLMATCH = 0x4000   # PyPy extension, for CPython >= 3.4
 
 
 def getlower(char_ord, flags):
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -89,6 +89,7 @@
     match_end = 0
     match_marks = None
     match_marks_flat = None
+    fullmatch_only = False
 
     def __init__(self, pattern, match_start, end, flags):
         # 'match_start' and 'end' must be known to be non-negative
@@ -527,7 +528,7 @@
             return
 
         elif op == OPCODE_SUCCESS:
-            if ctx.flags & rsre_char.SRE_FLAG_FULLMATCH:
+            if ctx.fullmatch_only:
                 if ptr != ctx.end:
                     return     # not a full match
             ctx.match_end = ptr
@@ -558,7 +559,11 @@
             # assert subpattern
             # <ASSERT> <0=skip> <1=back> <pattern>
             ptr1 = ptr - ctx.pat(ppos+1)
-            if ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None:
+            saved = ctx.fullmatch_only
+            ctx.fullmatch_only = False
+            stop = ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None
+            ctx.fullmatch_only = saved
+            if stop:
                 return
             marks = ctx.match_marks
             ppos += ctx.pat(ppos)
@@ -567,7 +572,12 @@
             # assert not subpattern
             # <ASSERT_NOT> <0=skip> <1=back> <pattern>
             ptr1 = ptr - ctx.pat(ppos+1)
-            if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) is not None:
+            saved = ctx.fullmatch_only
+            ctx.fullmatch_only = False
+            stop = (ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks)
+                                      is not None)
+            ctx.fullmatch_only = saved
+            if stop:
                 return
             ppos += ctx.pat(ppos)
 
@@ -1006,17 +1016,17 @@
     elif end > length: end = length
     return start, end
 
-def match(pattern, string, start=0, end=sys.maxint, flags=0):
+def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False):
     start, end = _adjust(start, end, len(string))
     ctx = StrMatchContext(pattern, string, start, end, flags)
+    ctx.fullmatch_only = fullmatch
     if match_context(ctx):
         return ctx
     else:
         return None
 
 def fullmatch(pattern, string, start=0, end=sys.maxint, flags=0):
-    return match(pattern, string, start, end,
-                 flags | rsre_char.SRE_FLAG_FULLMATCH)
+    return match(pattern, string, start, end, flags, fullmatch=True)
 
 def search(pattern, string, start=0, end=sys.maxint, flags=0):
     start, end = _adjust(start, end, len(string))
diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py
--- a/rpython/rlib/rsre/test/test_match.py
+++ b/rpython/rlib/rsre/test/test_match.py
@@ -293,3 +293,9 @@
         r = get_code(r"a((bp)*)c")
         match = rsre_core.fullmatch(r, "abpbpbpc")
         assert match.group(1) == "bpbpbp"
+
+    def test_fullmatch_assertion(self):
+        r = get_code(r"(?=a).b")
+        assert rsre_core.fullmatch(r, "ab")
+        r = get_code(r"(?!a)..")
+        assert not rsre_core.fullmatch(r, "ab")

From pypy.commits at gmail.com  Mon Aug 29 10:27:07 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 07:27:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: comment, test simplifiactions,
 added missing tests not included by merge
Message-ID: <57c4463b.c15e1c0a.42ba2.a989@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86700:520a2f63f5ec
Date: 2016-08-29 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/520a2f63f5ec/

Log:	comment, test simplifiactions, added missing tests not included by
	merge

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -32,6 +32,7 @@
         assert isinstance(buf, Buffer)
         self.buf = buf
         self._hash = -1
+        # private copies of format, shape, itemsize, ... on this class
         self.format = format
         self.itemsize = itemsize
         self.shape = shape
@@ -495,7 +496,7 @@
         if not newfmt:
             raise oefmt(space.w_RuntimeError,
                     "memoryview: internal error")
-        self.setformat(newfmt)
+        self.format = newfmt
         self.itemsize = itemsize
         self.ndim = 1
         self.shape = [buf.getlength() // buf.getitemsize()]
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -179,6 +179,54 @@
     def test_hex(self):
         assert memoryview(b"abc").hex() == u'616263'
 
+    def test_memoryview_cast(self):
+        m1 = memoryview(b'abcdefgh')
+        m2 = m1.cast('I')
+        m3 = m1.cast('h')
+        assert list(m1) == [97, 98, 99, 100, 101, 102, 103, 104]
+        assert list(m2) == [1684234849, 1751606885]
+        assert list(m3) == [25185, 25699, 26213, 26727]
+        assert m1[1] == 98
+        assert m2[1] == 1751606885
+        assert m3[1] == 25699
+        assert list(m3[1:3]) == [25699, 26213]
+        assert m3[1:3].tobytes() == b'cdef'
+        assert len(m2) == 2
+        assert len(m3) == 4
+        assert (m2[-2], m2[-1]) == (1684234849, 1751606885)
+        raises(IndexError, "m2[2]")
+        raises(IndexError, "m2[-3]")
+        assert list(m3[-99:3]) == [25185, 25699, 26213]
+        assert list(m3[1:99]) == [25699, 26213, 26727]
+        raises(IndexError, "m1[8]")
+        raises(IndexError, "m1[-9]")
+        assert m1[-8] == 97
+
+    def test_memoryview_cast_extended_slicing(self):
+        m1 = memoryview(b'abcdefgh')
+        m3 = m1.cast('h')
+        assert m3[1::2].tobytes() == b'cdgh'
+        assert m3[::2].tobytes() == b'abef'
+        assert m3[:2:2].tobytes() == b'ab'
+
+    def test_memoryview_cast_setitem(self):
+        data = bytearray(b'abcdefgh')
+        m1 = memoryview(data)
+        m2 = m1.cast('I')
+        m3 = m1.cast('h')
+        m1[2] = ord(b'C')
+        assert m2[0] == 1682137697
+        m3[1] = -9999
+        assert data == bytearray(bytes([97, 98, 241, 216, 101, 102, 103, 104]))
+        m3[1:3] = memoryview(b"pqrs").cast('h')
+        assert data == bytearray(b'abpqrsgh')
+
+    def test_memoryview_cast_setitem_extended_slicing(self):
+        data = bytearray(b'abcdefghij')
+        m3 = memoryview(data).cast('h')
+        m3[1:5:2] = memoryview(b"xyXY").cast('h')
+        assert data == bytearray(b'abxyefXYij')
+
 class MockBuffer(Buffer):
     def __init__(self, space, w_arr, w_dim, w_fmt, \
                  w_itemsize, w_strides, w_shape):
@@ -277,15 +325,8 @@
         assert view[-1,-1] == 11
         assert view[-3,-4] == 0
 
-        try:
-            view.__getitem__((2**63-1,0))
-            assert False, "must not succeed"
-        except IndexError: pass
-
-        try:
-            view.__getitem__((0, 0, 0))
-            assert False, "must not succeed"
-        except TypeError: pass
+        raises(IndexError, "view.__getitem__((2**63-1,0))")
+        raises(TypeError, "view.__getitem__((0, 0, 0))")
 
     def test_tuple_indexing_int(self):
         content = self.MockArray([ [[1],[2],[3]], [[4],[5],[6]] ],
@@ -298,6 +339,7 @@
     def test_cast_non_byte(self):
         empty = self.MockArray([], dim=1, fmt='i', size=4, strides=[1], shape=[1])
         view = memoryview(empty)
+        raises(TypeError, "view.cast('l')")
         try:
             view.cast('l')
             assert False, "i -> l not possible. buffer must be byte format"
@@ -313,14 +355,10 @@
         assert view.format == 'b'
         assert cview.format == 'i'
         #
-        #assert cview.cast('i').cast('b').cast('i').tolist() == []
+        assert cview.cast('i').cast('b').cast('i').tolist() == []
         #
         assert cview.format == 'i'
-        try:
-            cview.cast('i')
-            assert False, "cast must fail"
-        except TypeError:
-            pass
+        raises(TypeError, "cview.cast('i')")
 
     def test_cast_with_shape(self):
         empty = self.MockArray([1,0,2,0,3,0],
@@ -342,9 +380,4 @@
         view = memoryview(bytes)
         v = view.cast('h', shape=(3,2))
         assert v.tolist() == [[2,3],[4,5],[6,7]]
-        try:
-            v = view.cast('h', shape=(3,3))
-            assert False, "shape is too big for bytes"
-        except TypeError:
-            pass
-
+        raises(TypeError, "view.cast('h', shape=(3,3))")

From pypy.commits at gmail.com  Mon Aug 29 10:27:09 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 07:27:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: removed multiply where
 itemsize is already applied, simplification
Message-ID: <57c4463d.28eac20a.93a56.6eb6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86701:a011742662bf
Date: 2016-08-29 16:15 +0200
http://bitbucket.org/pypy/pypy/changeset/a011742662bf/

Log:	removed multiply where itemsize is already applied, simplification

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -225,15 +225,7 @@
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
         itemsize = self.getitemsize()
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            # XXX why? returns a memory view on int index if step == 0:
-            #    step = 1
-
-            if stop > self.buf.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
+        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
             if itemsize == 1:
                 ch = self.buf.getitem(start)
@@ -253,6 +245,16 @@
             raise oefmt(space.w_NotImplementedError,
                         "XXX extended slicing")
 
+    def _apply_itemsize(self, space, start, size, itemsize):
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+            stop  = start + size
+            # start & stop are now byte offset, thus use self.bug.getlength()
+            if stop > self.buf.getlength():
+                raise oefmt(space.w_IndexError, 'index out of range')
+        return start, stop, size
+
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
         if self.buf.readonly:
@@ -261,16 +263,7 @@
             raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         itemsize = self.getitemsize()
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            # XXX why? returns a memory view on int index if step == 0:
-            #    step = 1
-
-            # start & stop are now byte offset, thus use self.bug.getlength()
-            if stop > self.buf.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
+        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
             if itemsize == 1:
                 ch = getbytevalue(space, w_obj)
@@ -287,7 +280,7 @@
                 self.buf.setslice(start, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-            if value.getlength() != size * self.itemsize:
+            if value.getlength() != size:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
             self.buf.setslice(start, value.as_str())

From pypy.commits at gmail.com  Mon Aug 29 10:27:11 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 07:27:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: fixes,
 uncommented some test lines
Message-ID: <57c4463f.8aacc20a.f3495.6a1d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86702:e73551e958eb
Date: 2016-08-29 16:26 +0200
http://bitbucket.org/pypy/pypy/changeset/e73551e958eb/

Log:	fixes, uncommented some test lines

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -249,10 +249,12 @@
         if itemsize > 1:
             start *= itemsize
             size *= itemsize
-            stop  = start + size
-            # start & stop are now byte offset, thus use self.bug.getlength()
-            if stop > self.buf.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
+
+        stop  = start + size
+        # start & stop are now byte offset, thus use self.buf.getlength()
+        if stop > self.buf.getlength():
+            raise oefmt(space.w_IndexError, 'index out of range')
+
         return start, stop, size
 
     def descr_setitem(self, space, w_index, w_obj):
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -355,7 +355,7 @@
         assert view.format == 'b'
         assert cview.format == 'i'
         #
-        assert cview.cast('i').cast('b').cast('i').tolist() == []
+        assert cview.cast('b').cast('q').cast('b').tolist() == []
         #
         assert cview.format == 'i'
         raises(TypeError, "cview.cast('i')")
@@ -366,7 +366,7 @@
                     strides=[8], shape=[6])
         view = memoryview(empty)
         byteview = view.cast('b')
-        #assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]
+        assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]
         i32view = byteview.cast('i', shape=[1,3])
         assert i32view.format == 'i'
         assert i32view.itemsize == 4

From pypy.commits at gmail.com  Mon Aug 29 10:45:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 07:45:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: re.compile().__repr__
Message-ID: <57c44a76.6211c20a.7af0d.79f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86703:1e8e1e6907c8
Date: 2016-08-29 16:44 +0200
http://bitbucket.org/pypy/pypy/changeset/1e8e1e6907c8/

Log:	re.compile().__repr__

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -13,7 +13,7 @@
 #
 # Constants and exposed functions
 
-from rpython.rlib.rsre import rsre_core
+from rpython.rlib.rsre import rsre_core, rsre_char
 from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
 
 
@@ -92,6 +92,10 @@
 #
 # SRE_Pattern class
 
+FLAG_NAMES = ["re.TEMPLATE", "re.IGNORECASE", "re.LOCALE", "re.MULTILINE",
+              "re.DOTALL", "re.UNICODE", "re.VERBOSE", "re.DEBUG",
+              "re.ASCII"]
+
 class W_SRE_Pattern(W_Root):
     _immutable_fields_ = ["code", "flags", "num_groups", "w_groupindex"]
 
@@ -99,6 +103,43 @@
         space = self.space
         raise oefmt(space.w_TypeError, "cannot copy this pattern object")
 
+    def repr_w(self):
+        space = self.space
+        u = space.unicode_w(space.repr(self.w_pattern))
+        flag_items = []
+        flags = self.flags
+        if self.is_known_unicode():
+            if ((flags & (rsre_char.SRE_FLAG_LOCALE |
+                          rsre_char.SRE_FLAG_UNICODE |
+                          256))     # rsre_char.SRE_FLAG_ASCII
+                    == rsre_char.SRE_FLAG_UNICODE):
+                flags &= ~rsre_char.SRE_FLAG_UNICODE
+        for i, name in enumerate(FLAG_NAMES):
+            if flags & (1 << i):
+                flags -= (1 << i)
+                flag_items.append(name)
+        if flags != 0:
+            flag_items.append('0x%x' % flags)
+        if len(flag_items) == 0:
+            usep = u''
+            uflags = u''
+        else:
+            usep = u', '
+            uflags = u'|'.join([item.decode('latin-1') for item in flag_items])
+        return space.wrap(u're.compile(%s%s%s)' % (u, usep, uflags))
+
+    def is_known_bytes(self):
+        space = self.space
+        if space.is_none(self.w_pattern):
+            return False
+        return not space.isinstance_w(self.w_pattern, space.w_unicode)
+
+    def is_known_unicode(self):
+        space = self.space
+        if space.is_none(self.w_pattern):
+            return False
+        return space.isinstance_w(self.w_pattern, space.w_unicode)
+
     def make_ctx(self, w_string, pos=0, endpos=sys.maxint, flags=0):
         """Make a StrMatchContext, BufMatchContext or a UnicodeMatchContext for
         searching in the given w_string object."""
@@ -110,8 +151,7 @@
         flags = self.flags | flags
         if space.isinstance_w(w_string, space.w_unicode):
             unicodestr = space.unicode_w(w_string)
-            if not (space.is_none(self.w_pattern) or
-                    space.isinstance_w(self.w_pattern, space.w_unicode)):
+            if self.is_known_bytes():
                 raise oefmt(space.w_TypeError,
                             "can't use a bytes pattern on a string-like "
                             "object")
@@ -122,8 +162,7 @@
             return rsre_core.UnicodeMatchContext(self.code, unicodestr,
                                                  pos, endpos, flags)
         elif space.isinstance_w(w_string, space.w_str):
-            if (not space.is_none(self.w_pattern) and
-                space.isinstance_w(self.w_pattern, space.w_unicode)):
+            if self.is_known_unicode():
                 raise oefmt(space.w_TypeError,
                             "can't use a string pattern on a bytes-like "
                             "object")
@@ -136,8 +175,7 @@
                                              pos, endpos, flags)
         else:
             buf = space.readbuf_w(w_string)
-            if (not space.is_none(self.w_pattern) and
-                space.isinstance_w(self.w_pattern, space.w_unicode)):
+            if self.is_known_unicode():
                 raise oefmt(space.w_TypeError,
                             "can't use a string pattern on a bytes-like "
                             "object")
@@ -422,6 +460,7 @@
     __new__      = interp2app(SRE_Pattern__new__),
     __copy__     = interp2app(W_SRE_Pattern.cannot_copy_w),
     __deepcopy__ = interp2app(W_SRE_Pattern.cannot_copy_w),
+    __repr__     = interp2app(W_SRE_Pattern.repr_w),
     __weakref__  = make_weakref_descr(W_SRE_Pattern),
     findall      = interp2app(W_SRE_Pattern.findall_w),
     finditer     = interp2app(W_SRE_Pattern.finditer_w),
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -123,6 +123,15 @@
         assert re.fullmatch(r"ab*c", "abbbcdef") is None
         assert re.fullmatch(r"ab*c", "abbbc") is not None
 
+    def test_repr(self):
+        import re
+        r = re.compile(r'f(o"\d)', 0)
+        assert repr(r) == (
+            r"""re.compile('f(o"\\d)')""")
+        r = re.compile(r'f(o"\d)', re.IGNORECASE|re.DOTALL|re.VERBOSE)
+        assert repr(r) == (
+            r"""re.compile('f(o"\\d)', re.IGNORECASE|re.DOTALL|re.VERBOSE)""")
+
 
 class AppTestSreMatch:
     spaceconfig = dict(usemodules=('array', ))

From pypy.commits at gmail.com  Mon Aug 29 11:17:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 08:17:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: hg merge py3.5
Message-ID: <57c45226.05d71c0a.351b2.5b3d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86704:fc302e9992ff
Date: 2016-08-29 17:13 +0200
http://bitbucket.org/pypy/pypy/changeset/fc302e9992ff/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 6469 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,6 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
diff --git a/lib-python/3/importlib/_bootstrap_external.py b/lib-python/3/importlib/_bootstrap_external.py
--- a/lib-python/3/importlib/_bootstrap_external.py
+++ b/lib-python/3/importlib/_bootstrap_external.py
@@ -228,7 +228,14 @@
 # longer be understood by older implementations of the eval loop (usually
 # due to the addition of new opcodes).
 
-MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n'
+# MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n'
+#
+# PyPy change: the MAGIC_NUMBER is defined in
+# pypy/interpreter/pycode.py, 'default_magic'.  It is based on a number
+# different than CPython's, always < 3000.  We get the 4-bytes string
+# here via a hack: MAGIC_NUMBER is set in the module from
+# module/_frozen_importlib/__init__.py before the module is executed.
+
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -166,8 +166,8 @@
         else:
             return self.value
 
-    def __buffer__(self):
-        return memoryview(self._buffer)
+    def __buffer__(self, flags):
+        return buffer(self._buffer)
 
     def _get_b_base(self):
         try:
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -554,6 +554,9 @@
     def putwin(self, filep):
         # filestar = ffi.new("FILE *", filep)
         return _check_ERR(lib.putwin(self._win, filep), "putwin")
+        # XXX CPython 3.5 says: We have to simulate this by writing to
+        # a temporary FILE*, then reading back, then writing to the
+        # argument stream.
 
     def redrawln(self, beg, num):
         return _check_ERR(lib.wredrawln(self._win, beg, num), "redrawln")
@@ -704,6 +707,7 @@
 
 
 def getwin(filep):
+    # XXX CPython 3.5: there's logic to use a temp file instead
     return Window(_check_NULL(lib.getwin(filep)))
 
 
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -49,9 +49,11 @@
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
-    # some parts of code.py are copied here because it seems to be impossible
+    # some parts of code.py are copied here because it was impossible
     # to start an interactive console without printing at least one line
-    # of banner
+    # of banner.  This was fixed in 3.4; but then from 3.6 it prints a
+    # line when exiting.  This can be disabled too---by passing an argument
+    # that doesn't exist in <= 3.5.  So, too much mess: just copy the code.
     more = 0
     while 1:
         try:
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,219 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. 
+
+XXXXX MORE ???
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Support for memoryview attributes (format, itemsize, ...) which also
+    adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,146 +1,8 @@
-=========================
-What's new in PyPy2.7 5.3+
-=========================
+==========================
+What's new in PyPy2.7 5.4+
+==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
 
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -1,5 +1,5 @@
 =========================
-What's new in PyPy2.7 5.3+
+What's new in PyPy2.7 5.4
 =========================
 
 .. this is a revision shortly after release-pypy2.7-v5.3
@@ -144,3 +144,22 @@
 ``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
 Previously it returned what looked like a regular dict object (but it
 was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -432,7 +432,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(w_decorator_list)
         _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        _returns = expr.from_object(space, w_returns) if w_returns is not None else None
+        _returns = expr.from_object(space, w_returns)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return FunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset)
@@ -508,7 +508,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(w_decorator_list)
         _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
-        _returns = expr.from_object(space, w_returns) if w_returns is not None else None
+        _returns = expr.from_object(space, w_returns)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return AsyncFunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset)
@@ -630,7 +630,7 @@
         w_value = get_field(space, w_node, 'value', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _value = expr.from_object(space, w_value) if w_value is not None else None
+        _value = expr.from_object(space, w_value)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Return(_value, _lineno, _col_offset)
@@ -1190,8 +1190,8 @@
         w_cause = get_field(space, w_node, 'cause', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _exc = expr.from_object(space, w_exc) if w_exc is not None else None
-        _cause = expr.from_object(space, w_cause) if w_cause is not None else None
+        _exc = expr.from_object(space, w_exc)
+        _cause = expr.from_object(space, w_cause)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Raise(_exc, _cause, _lineno, _col_offset)
@@ -1314,7 +1314,7 @@
         _test = expr.from_object(space, w_test)
         if _test is None:
             raise_required_value(space, w_node, 'test')
-        _msg = expr.from_object(space, w_msg) if w_msg is not None else None
+        _msg = expr.from_object(space, w_msg)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Assert(_test, _msg, _lineno, _col_offset)
@@ -2312,7 +2312,7 @@
         w_value = get_field(space, w_node, 'value', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _value = expr.from_object(space, w_value) if w_value is not None else None
+        _value = expr.from_object(space, w_value)
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Yield(_value, _lineno, _col_offset)
@@ -3101,9 +3101,9 @@
         w_lower = get_field(space, w_node, 'lower', True)
         w_upper = get_field(space, w_node, 'upper', True)
         w_step = get_field(space, w_node, 'step', True)
-        _lower = expr.from_object(space, w_lower) if w_lower is not None else None
-        _upper = expr.from_object(space, w_upper) if w_upper is not None else None
-        _step = expr.from_object(space, w_step) if w_step is not None else None
+        _lower = expr.from_object(space, w_lower)
+        _upper = expr.from_object(space, w_upper)
+        _step = expr.from_object(space, w_step)
         return Slice(_lower, _upper, _step)
 
 State.ast_type('Slice', 'slice', ['lower', 'upper', 'step'])
@@ -3583,7 +3583,7 @@
         w_body = get_field(space, w_node, 'body', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _type = expr.from_object(space, w_type) if w_type is not None else None
+        _type = expr.from_object(space, w_type)
         _name = space.str_or_None_w(w_name)
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
@@ -3664,12 +3664,12 @@
         w_defaults = get_field(space, w_node, 'defaults', False)
         args_w = space.unpackiterable(w_args)
         _args = [arg.from_object(space, w_item) for w_item in args_w]
-        _vararg = arg.from_object(space, w_vararg) if w_vararg is not None else None
+        _vararg = arg.from_object(space, w_vararg) if not space.is_w(w_vararg, space.w_None) else None
         kwonlyargs_w = space.unpackiterable(w_kwonlyargs)
         _kwonlyargs = [arg.from_object(space, w_item) for w_item in kwonlyargs_w]
         kw_defaults_w = space.unpackiterable(w_kw_defaults)
         _kw_defaults = [expr.from_object(space, w_item) for w_item in kw_defaults_w]
-        _kwarg = arg.from_object(space, w_kwarg) if w_kwarg is not None else None
+        _kwarg = arg.from_object(space, w_kwarg) if not space.is_w(w_kwarg, space.w_None) else None
         defaults_w = space.unpackiterable(w_defaults)
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwonlyargs, _kw_defaults, _kwarg, _defaults)
@@ -3705,7 +3705,7 @@
         _arg = space.identifier_w(w_arg)
         if _arg is None:
             raise_required_value(space, w_node, 'arg')
-        _annotation = expr.from_object(space, w_annotation) if w_annotation is not None else None
+        _annotation = expr.from_object(space, w_annotation)
         return arg(_arg, _annotation)
 
 State.ast_type('arg', 'AST', ['arg', 'annotation'])
@@ -3805,7 +3805,7 @@
         _context_expr = expr.from_object(space, w_context_expr)
         if _context_expr is None:
             raise_required_value(space, w_node, 'context_expr')
-        _optional_vars = expr.from_object(space, w_optional_vars) if w_optional_vars is not None else None
+        _optional_vars = expr.from_object(space, w_optional_vars)
         return withitem(_context_expr, _optional_vars)
 
 State.ast_type('withitem', 'AST', ['context_expr', 'optional_vars'])
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -4,12 +4,20 @@
 from pypy.interpreter.pyparser.pygram import syms, tokens
 from pypy.interpreter.pyparser.error import SyntaxError
 from pypy.interpreter.pyparser import parsestring
-from rpython.rlib.objectmodel import always_inline
+from rpython.rlib.objectmodel import always_inline, we_are_translated
 
 
 def ast_from_node(space, node, compile_info):
     """Turn a parse tree, node, to AST."""
-    return ASTBuilder(space, node, compile_info).build_ast()
+    ast = ASTBuilder(space, node, compile_info).build_ast()
+    #
+    # When we are not translated, we send this ast to validate_ast.
+    # The goal is to check that validate_ast doesn't crash on valid
+    # asts, at least.
+    if not we_are_translated():
+        from pypy.interpreter.astcompiler import validate
+        validate.validate_ast(space, ast)
+    return ast
 
 
 augassign_operator_map = {
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1234,8 +1234,7 @@
         if d.values:
             for i in range(len(d.values)):
                 key = d.keys[i]
-                if key is None:
-                    is_unpacking = True
+                is_unpacking = key is None
                 if elements == 0xFFFF or (elements and is_unpacking):
                     self.emit_op_arg(ops.BUILD_MAP, elements)
                     containers += 1
@@ -1244,8 +1243,14 @@
                     d.values[i].walkabout(self)
                     containers += 1
                 else:
+                    # TODO: key.walkabout has to be called before d.values.walkabout
+                    # that would fix the error "keywords must be strings"
+                    # for some reason the keys and values seem to be in reverse order
+                    # in some cases, so another error has to be fixed in order for
+                    # this to work, otherwise it breaks everything
+                    # after fix: remove dirty fixes in pyopcode
                     d.values[i].walkabout(self)
-                    d.keys[i].walkabout(self)
+                    key.walkabout(self)
                     elements += 1
         if elements or containers == 0:
             self.emit_op_arg(ops.BUILD_MAP, elements)
@@ -1257,7 +1262,7 @@
             oparg = min(containers, 255)
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
-            is_unpacking = 0
+            is_unpacking = False
     
     def visit_Set(self, s):
         self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -16,12 +16,14 @@
     return codegen.compile_ast(space, ast, info)
 
 def generate_function_code(expr, space):
+    from pypy.interpreter.astcompiler.ast import FunctionDef
     p = pyparse.PythonParser(space)
     info = pyparse.CompileInfo("<test>", 'exec')
     cst = p.parse_source(expr, info)
     ast = astbuilder.ast_from_node(space, cst, info)
     function_ast = optimize.optimize_ast(space, ast.body[0], info)
     function_ast = ast.body[0]
+    assert isinstance(function_ast, FunctionDef)
     symbols = symtable.SymtableBuilder(space, ast, info)
     generator = codegen.FunctionCodeGenerator(
         space, 'function', function_ast, 1, symbols, info, qualname='function')
@@ -864,9 +866,10 @@
         with a: pass
         with a: pass
         with a: pass
+        with a: pass
         """
         code = compile_with_astcompiler(source, 'exec', self.space)
-        assert code.co_stacksize == 5
+        assert code.co_stacksize == 6  # i.e. <= 7, there is no systematic leak
 
     def test_stackeffect_bug5(self):
         source = """if 1:
@@ -1345,7 +1348,9 @@
             assert ops.BINARY_POWER not in counts
 
     def test_call_function_var(self):
-        source = """call(*me)"""
+        source = """def f():
+            call(*me)
+        """
         code, blocks = generate_function_code(source, self.space)
         # there is a stack computation error
         assert blocks[0].instructions[3].arg == 0
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -160,7 +160,17 @@
         else:
             extractor = "%s.from_object(space, %s)" % (field.type, value)
             if field.opt:
-                extractor += " if %s is not None else None" % (value,)
+                if field.type == 'expr':
+                    # the expr.from_object() method should accept w_None and
+                    # return None; nothing more to do here
+                    pass
+                elif field.type == 'arg':
+                    # the method arg.from_object() doesn't accept w_None
+                    extractor += (
+                        ' if not space.is_w(%s, space.w_None) else None'
+                        % (value,))
+                else:
+                    raise NotImplementedError(field.type)
             return extractor
 
     def get_field_converter(self, field):
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -176,6 +176,13 @@
         if node.returns:
             self._validate_expr(node.returns)
 
+    def visit_AsyncFunctionDef(self, node):
+        self._validate_body(node.body, "AsyncFunctionDef")
+        node.args.walkabout(self)
+        self._validate_exprs(node.decorator_list)
+        if node.returns:
+            self._validate_expr(node.returns)
+
     def visit_keyword(self, node):
         self._validate_expr(node.value)
 
@@ -193,6 +200,9 @@
         if node.value:
             self._validate_expr(node.value)
 
+    def visit_Await(self, node):
+        self._validate_expr(node.value)
+
     def visit_Delete(self, node):
         self._validate_nonempty_seq(node.targets, "targets", "Delete")
         self._validate_exprs(node.targets, ast.Del)
@@ -212,6 +222,12 @@
         self._validate_body(node.body, "For")
         self._validate_stmts(node.orelse)
 
+    def visit_AsyncFor(self, node):
+        self._validate_expr(node.target, ast.Store)
+        self._validate_expr(node.iter)
+        self._validate_body(node.body, "AsyncFor")
+        self._validate_stmts(node.orelse)
+
     def visit_While(self, node):
         self._validate_expr(node.test)
         self._validate_body(node.body, "While")
@@ -232,6 +248,11 @@
         self.visit_sequence(node.items)
         self._validate_body(node.body, "With")
 
+    def visit_AsyncWith(self, node):
+        self._validate_nonempty_seq(node.items, "items", "AsyncWith")
+        self.visit_sequence(node.items)
+        self._validate_body(node.body, "AsyncWith")
+
     def visit_Raise(self, node):
         if node.exc:
             self._validate_expr(node.exc)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -837,13 +837,18 @@
             self.interned_strings.set(u, w_s1)
         return w_s1
 
-    def is_interned_str(self, s):
-        """Assumes an identifier (utf-8 encoded str)"""
+    def get_interned_str(self, s):
+        """Assumes an identifier (utf-8 encoded str).  Returns None if
+        the identifier is not interned, or not a valid utf-8 string at all.
+        """
         # interface for marshal_impl
         if not we_are_translated():
             assert type(s) is str
-        u = s.decode('utf-8')
-        return self.interned_strings.get(u) is not None
+        try:
+            u = s.decode('utf-8')
+        except UnicodeDecodeError:
+            return None
+        return self.interned_strings.get(u)   # may be None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
@@ -942,8 +947,8 @@
             idx += 1
         if idx < expected_length:
             raise oefmt(self.w_ValueError,
-                        "need more than %d value%s to unpack",
-                        idx, "" if idx == 1 else "s")
+                        "not enough values to unpack (expected %d, got %d)",
+                        expected_length, idx)
         return items
 
     def unpackiterable_unroll(self, w_iterable, expected_length):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -35,9 +35,10 @@
 
 # we compute the magic number in a similar way to CPython, but we use a
 # different value for the highest 16 bits. Bump pypy_incremental_magic every
-# time you make pyc files incompatible
+# time you make pyc files incompatible.  This value ends up in the frozen
+# importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 64 # bump it by 16
+pypy_incremental_magic = 80 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -820,13 +820,9 @@
         itemcount = len(items)
         count = left + right
         if count > itemcount:
-            if count == 1:
-                plural = ''
-            else:
-                plural = 's'
             raise oefmt(self.space.w_ValueError,
-                        "need more than %d value%s to unpack",
-                        itemcount, plural)
+                        "not enough values to unpack (expected at least %d, got %d)",
+                        count, itemcount)
         right = itemcount - right
         assert right >= 0
         # push values in reverse order
@@ -1202,7 +1198,7 @@
                 self.settopvalue(self.space.w_None)
 
     @jit.unroll_safe
-    def call_function(self, oparg, w_star=None, w_starstar=None):
+    def call_function(self, oparg, w_starstar=None, has_vararg=False):
         n_arguments = oparg & 0xff
         n_keywords = (oparg>>8) & 0xff
         if n_keywords:
@@ -1214,20 +1210,16 @@
                     break
                 w_value = self.popvalue()
                 w_key = self.popvalue()
-                # temporary (dirty) fix: if star-arg occurs after kwarg,
-                # arg order is reversed on stack
-                from pypy.objspace.std.listobject import W_ListObject
-                if isinstance(w_key, W_ListObject):
-                    w_key_temp = w_key
-                    w_key = w_value
-                    w_value = w_star
-                    w_star = w_key_temp
                 key = self.space.identifier_w(w_key)
                 keywords[n_keywords] = key
                 keywords_w[n_keywords] = w_value
         else:
             keywords = None
             keywords_w = None
+        if has_vararg:
+            w_star = self.popvalue()
+        else:
+            w_star = None
         arguments = self.popvalues(n_arguments)
         args = self.argument_factory(arguments, keywords, keywords_w, w_star,
                                      w_starstar)
@@ -1256,17 +1248,15 @@
             self.call_function(oparg)
 
     def CALL_FUNCTION_VAR(self, oparg, next_instr):
-        w_varargs = self.popvalue()
-        self.call_function(oparg, w_varargs)
+        self.call_function(oparg, has_vararg=True)
 
     def CALL_FUNCTION_KW(self, oparg, next_instr):
         w_varkw = self.popvalue()
-        self.call_function(oparg, None, w_varkw)
+        self.call_function(oparg, w_varkw)
 
     def CALL_FUNCTION_VAR_KW(self, oparg, next_instr):
         w_varkw = self.popvalue()
-        w_varargs = self.popvalue()
-        self.call_function(oparg, w_varargs, w_varkw)
+        self.call_function(oparg, w_varkw, has_vararg=True)
 
     @jit.unroll_safe
     def _make_function(self, oparg, freevars=None):
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -214,6 +214,15 @@
         assert self.codetest(code, 'g', [12, {}]) ==    ()
         assert self.codetest(code, 'g', [12, {3:1}]) == (3,)
 
+    def test_star_arg_after_keyword_arg(self):
+        code = '''
+            def f(a, b):
+                return a - b
+            def g(a, b):
+                return f(b=b, *(a,))
+        '''
+        assert self.codetest(code, 'g', [40, 2]) == 38
+
     def test_closure(self):
         code = '''
             def f(x, y):
@@ -458,7 +467,7 @@
         try:
             a, *b, c, d, e = Seq()
         except ValueError as e:
-            assert str(e) == "need more than 3 values to unpack"
+            assert str(e) == "not enough values to unpack (expected at least 4, got 3)"
         else:
             assert False, "Expected ValueError"
             """
diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py
deleted file mode 100644
--- a/pypy/module/_file/readinto.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import sys, errno
-from rpython.rlib import rposix
-from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.lltypesystem import lltype, rffi
-from pypy.module._file.interp_file import is_wouldblock_error, signal_checker
-
-_WIN32 = sys.platform.startswith('win')
-UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
-
-os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read',
-                          [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
-                          rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO)
-
-
-def direct_readinto(self, w_rwbuffer):
-    rwbuffer = self.space.writebuf_w(w_rwbuffer)
-    stream = self.getstream()
-    size = rwbuffer.getlength()
-    target_address = lltype.nullptr(rffi.CCHARP.TO)
-    fd = -1
-    target_pos = 0
-
-    if size > 64:
-        try:
-            target_address = rwbuffer.get_raw_address()
-        except ValueError:
-            pass
-        else:
-            fd = stream.try_to_find_file_descriptor()
-
-    if fd < 0 or not target_address:
-        # fall-back
-        MAX_PART = 1024 * 1024    # 1 MB
-        while size > MAX_PART:
-            data = self.direct_read(MAX_PART)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-            if len(data) != MAX_PART:
-                break
-        else:
-            data = self.direct_read(size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-
-    else:
-        # optimized case: reading more than 64 bytes into a rwbuffer
-        # with a valid raw address
-        self.check_readable()
-
-        # first "read" the part that is already sitting in buffers, if any
-        initial_size = min(size, stream.count_buffered_bytes())
-        if initial_size > 0:
-            data = stream.read(initial_size)
-            rwbuffer.setslice(target_pos, data)
-            target_pos += len(data)
-            size -= len(data)
-
-        # then call os_read() to get the rest
-        if size > 0:
-            stream.flush()
-            while True:
-                got = os_read(fd, rffi.ptradd(target_address, target_pos), size)
-                got = rffi.cast(lltype.Signed, got)
-                if got > 0:
-                    target_pos += got
-                    size -= got
-                    if size <= 0:
-                        break
-                elif got == 0:
-                    break
-                else:
-                    err = rposix.get_saved_errno()
-                    if err == errno.EINTR:
-                        signal_checker(self.space)()
-                        continue
-                    if is_wouldblock_error(err) and target_pos > 0:
-                        break
-                    raise OSError(err, "read error")
-            keepalive_until_here(rwbuffer)
-
-    return self.space.wrap(target_pos)
diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -19,7 +19,8 @@
         with open(os.path.join(lib_python, 'importlib', name + '.py')) as fp:
             source = fp.read()
         pathname = "<frozen importlib.%s>" % name
-        code_w = Module._cached_compile(space, source, pathname, 'exec', 0)
+        code_w = Module._cached_compile(space, name, source,
+                                        pathname, 'exec', 0)
         space.setitem(w_dict, space.wrap('__name__'), w_name)
         space.setitem(w_dict, space.wrap('__builtins__'),
                       space.wrap(space.builtin))
@@ -27,10 +28,15 @@
 
     def install(self):
         """NOT_RPYTHON"""
+        from pypy.module.imp import interp_imp
+
         super(Module, self).install()
         space = self.space
         # "import importlib/_boostrap_external.py"
         w_mod = Module(space, space.wrap("_frozen_importlib_external"))
+        # hack: inject MAGIC_NUMBER into this module's dict
+        space.setattr(w_mod, space.wrap('MAGIC_NUMBER'),
+                      interp_imp.get_magic(space))
         self._compile_bootstrap_module(
             space, '_bootstrap_external', w_mod.w_name, w_mod.w_dict)
         space.sys.setmodule(w_mod)
@@ -43,11 +49,13 @@
         self.w_import = space.wrap(interp_import.import_with_frames_removed)
 
     @staticmethod
-    def _cached_compile(space, source, *args):
+    def _cached_compile(space, name, source, *args):
         from rpython.config.translationoption import CACHE_DIR
         from pypy.module.marshal import interp_marshal
+        from pypy.interpreter.pycode import default_magic
 
-        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_bootstrap')
+        cachename = os.path.join(CACHE_DIR, 'frozen_importlib_%d%s' % (
+            default_magic, name))
         try:
             if space.config.translating:
                 raise IOError("don't use the cache when translating pypy")
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -4,6 +4,7 @@
     OperationError, oefmt, wrap_oserror, wrap_oserror2)
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import rposix
 from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC, O_EXCL
 import sys, os, stat, errno
 from pypy.module._io.interp_iobase import W_RawIOBase, convert_size
@@ -29,6 +30,7 @@
 
 O_BINARY = getattr(os, "O_BINARY", 0)
 O_APPEND = getattr(os, "O_APPEND", 0)
+_open_inhcache = rposix.SetNonInheritableCache()
 
 def _bad_mode(space):
     raise oefmt(space.w_ValueError,
@@ -139,6 +141,7 @@
 
     @unwrap_spec(mode=str, closefd=int)
     def descr_init(self, space, w_name, mode='r', closefd=True, w_opener=None):
+        self._close(space)
         if space.isinstance_w(w_name, space.w_float):
             raise oefmt(space.w_TypeError,
                         "integer argument expected, got float")
@@ -153,6 +156,8 @@
                 raise oefmt(space.w_ValueError, "negative file descriptor")
 
         self.readable, self.writable, self.created, self.appending, flags = decode_mode(space, mode)
+        if rposix.O_CLOEXEC is not None:
+            flags |= rposix.O_CLOEXEC
 
         fd_is_own = False
         try:
@@ -171,8 +176,7 @@
                     raise oefmt(space.w_ValueError,
                                 "Cannot use closefd=False with file name")
 
-                from pypy.module.posix.interp_posix import (
-                    dispatch_filename, rposix)
+                from pypy.module.posix.interp_posix import dispatch_filename
                 try:
                     self.fd = dispatch_filename(rposix.open)(
                         space, w_name, flags, 0666)
@@ -181,6 +185,11 @@
                                         exception_name='w_IOError')
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    try:
+                        _open_inhcache.set_non_inheritable(self.fd)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
             else:
                 w_fd = space.call_function(w_opener, w_name, space.wrap(flags))
                 try:
@@ -192,6 +201,11 @@
                                 "expected integer from opener")
                 finally:
                     fd_is_own = True
+                if not rposix._WIN32:
+                    try:
+                        rposix.set_inheritable(self.fd, False)
+                    except OSError as e:
+                        raise wrap_oserror2(space, e, w_name)
 
             self._dircheck(space, w_name)
             space.setattr(self, space.wrap("name"), w_name)
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -246,6 +246,33 @@
             assert f.mode == 'xb'
         raises(FileExistsError, _io.FileIO, filename, 'x')
 
+    def test_non_inheritable(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        assert posix.get_inheritable(f.fileno()) == False
+        f.close()
+
+    def test_FileIO_fd_does_not_change_inheritable(self):
+        import _io, posix
+        fd1, fd2 = posix.pipe()
+        posix.set_inheritable(fd1, True)
+        posix.set_inheritable(fd2, False)
+        f1 = _io.FileIO(fd1, 'r')
+        f2 = _io.FileIO(fd2, 'w')
+        assert posix.get_inheritable(fd1) == True
+        assert posix.get_inheritable(fd2) == False
+        f1.close()
+        f2.close()
+
+    def test_close_upon_reinit(self):
+        import _io, posix
+        f = _io.FileIO(self.tmpfile, 'r')
+        fd1 = f.fileno()
+        f.__init__(self.tmpfile, 'w')
+        fd2 = f.fileno()
+        if fd1 != fd2:
+            raises(OSError, posix.close, fd1)
+
 
 def test_flush_at_exit():
     from pypy import conftest
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.c b/pypy/module/_posixsubprocess/_posixsubprocess.c
--- a/pypy/module/_posixsubprocess/_posixsubprocess.c
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.c
@@ -106,6 +106,30 @@
 }
 
 
+RPY_EXTERN
+int rpy_set_inheritable(int fd, int inheritable);   /* rposix.py */
+
+static int
+make_inheritable(long *py_fds_to_keep, ssize_t num_fds_to_keep,
+                 int errpipe_write)
+{
+    long i;
+
+    for (i = 0; i < num_fds_to_keep; ++i) {
+        long fd = py_fds_to_keep[i];
+        if (fd == errpipe_write) {
+            /* errpipe_write is part of py_fds_to_keep. It must be closed at
+               exec(), but kept open in the child process until exec() is
+               called. */
+            continue;
+        }
+        if (rpy_set_inheritable((int)fd, 1) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+
 /* Close all file descriptors in the range start_fd inclusive to
  * end_fd exclusive except for those in py_fds_to_keep.  If the
  * range defined by [start_fd, end_fd) is large this will take a
@@ -329,6 +353,9 @@
     /* Buffer large enough to hold a hex integer.  We can't malloc. */
     char hex_errno[sizeof(saved_errno)*2+1];
 
+    if (make_inheritable(py_fds_to_keep, num_fds_to_keep, errpipe_write) < 0)
+        goto error;
+
     /* Close parent's pipe ends. */
     if (p2cwrite != -1) {
         POSIX_CALL(close(p2cwrite));
@@ -352,26 +379,25 @@
        dup2() removes the CLOEXEC flag but we must do it ourselves if dup2()
        would be a no-op (issue #10806). */
     if (p2cread == 0) {
-        int old = fcntl(p2cread, F_GETFD);
-        if (old != -1)
-            fcntl(p2cread, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (p2cread != -1) {
+        if (rpy_set_inheritable(p2cread, 1) < 0)
+            goto error;
+    }
+    else if (p2cread != -1)
         POSIX_CALL(dup2(p2cread, 0));  /* stdin */
+
+    if (c2pwrite == 1) {
+        if (rpy_set_inheritable(c2pwrite, 1) < 0)
+            goto error;
     }
-    if (c2pwrite == 1) {
-        int old = fcntl(c2pwrite, F_GETFD);
-        if (old != -1)
-            fcntl(c2pwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (c2pwrite != -1) {
+    else if (c2pwrite != -1)
         POSIX_CALL(dup2(c2pwrite, 1));  /* stdout */
+
+    if (errwrite == 2) {
+        if (rpy_set_inheritable(errwrite, 1) < 0)
+            goto error;
     }
-    if (errwrite == 2) {
-        int old = fcntl(errwrite, F_GETFD);
-        if (old != -1)
-            fcntl(errwrite, F_SETFD, old & ~FD_CLOEXEC);
-    } else if (errwrite != -1) {
+    else if (errwrite != -1)
         POSIX_CALL(dup2(errwrite, 2));  /* stderr */
-    }
 
     /* Close pipe fds.  Make sure we don't close the same fd more than */
     /* once, or standard fds. */
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.h b/pypy/module/_posixsubprocess/_posixsubprocess.h
--- a/pypy/module/_posixsubprocess/_posixsubprocess.h
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.h
@@ -1,3 +1,4 @@
+#include <unistd.h>    /* for ssize_t */
 #include "src/precommondefs.h"
 
 RPY_EXTERN void
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -5,6 +5,7 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib import rposix
 
 from pypy.interpreter.error import (
     OperationError, exception_from_saved_errno, oefmt, wrap_oserror)
@@ -36,6 +37,7 @@
     compile_extra.append("-DHAVE_SETSID")
 
 eci = eci.merge(
+    rposix.eci_inheritable,
     ExternalCompilationInfo(
         compile_extra=compile_extra))
 
diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py
--- a/pypy/module/_posixsubprocess/test/test_subprocess.py
+++ b/pypy/module/_posixsubprocess/test/test_subprocess.py
@@ -75,3 +75,18 @@
         n = 1
         raises(OverflowError, _posixsubprocess.fork_exec,
                1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17)
+
+    def test_pass_fds_make_inheritable(self):
+        import subprocess, posix
+
+        fd1, fd2 = posix.pipe()
+        assert posix.get_inheritable(fd1) is False
+        assert posix.get_inheritable(fd2) is False
+
+        subprocess.check_call(['/usr/bin/env', 'python', '-c',
+                               'import os;os.write(%d,b"K")' % fd2],
+                              close_fds=True, pass_fds=[fd2])
+        res = posix.read(fd1, 1)
+        assert res == b"K"
+        posix.close(fd1)
+        posix.close(fd2)
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -142,21 +142,11 @@
 
 @unwrap_spec(fd=int)
 def dup(space, fd):
-    newfd = rsocket.dup(fd)
-    return space.wrap(newfd)
-
- at unwrap_spec(fd=int, family=int, type=int, proto=int)
-def fromfd(space, fd, family, type, proto=0):
-    """fromfd(fd, family, type[, proto]) -> socket object
-
-    Create a socket object from the given file descriptor.
-    The remaining arguments are the same as for socket().
-    """
     try:
-        sock = rsocket.fromfd(fd, family, type, proto)
+        newfd = rsocket.dup(fd, inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(W_Socket(space, sock))
+    return space.wrap(newfd)
 
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
@@ -170,7 +160,8 @@
     AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
     """
     try:
-        sock1, sock2 = rsocket.socketpair(family, type, proto)
+        sock1, sock2 = rsocket.socketpair(family, type, proto,
+                                          inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
     return space.newtuple([
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -177,7 +177,7 @@
                 sock = RSocket(family, type, proto,
                                fd=space.c_filedescriptor_w(w_fileno))
             else:
-                sock = RSocket(family, type, proto)
+                sock = RSocket(family, type, proto, inheritable=False)
             W_Socket.__init__(self, space, sock)
         except SocketError as e:
             raise converted_error(space, e)
@@ -228,7 +228,7 @@
         For IP sockets, the address info is a pair (hostaddr, port).
         """
         try:
-            fd, addr = self.sock.accept()
+            fd, addr = self.sock.accept(inheritable=False)
             return space.newtuple([space.wrap(fd),
                                    addr_as_object(addr, fd, space)])
         except SocketError as e:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -546,11 +546,19 @@
         s.ioctl(_socket.SIO_KEEPALIVE_VALS, (1, 100, 100))
 
     def test_dup(self):
-        import _socket as socket
+        import _socket as socket, posix
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.bind(('localhost', 0))
         fd = socket.dup(s.fileno())
         assert s.fileno() != fd
+        assert posix.get_inheritable(s.fileno()) is False
+        assert posix.get_inheritable(fd) is False
+        posix.close(fd)
+        s.close()
+
+    def test_dup_error(self):
+        import _socket
+        raises(_socket.error, _socket.dup, 123456)
 
     def test_buffer(self):
         # Test that send/sendall/sendto accept a buffer as arg
@@ -648,6 +656,26 @@
         assert len(w) == 1, [str(warning) for warning in w]
         assert r in str(w[0])
 
+    def test_invalid_fd(self):
+        import _socket
+        raises(ValueError, _socket.socket, fileno=-1)
+
+    def test_socket_non_inheritable(self):
+        import _socket, posix
+        s1 = _socket.socket()
+        assert posix.get_inheritable(s1.fileno()) is False
+        s1.close()
+
+    def test_socketpair_non_inheritable(self):
+        import _socket, posix
+        if not hasattr(_socket, 'socketpair'):
+            skip("no socketpair")
+        s1, s2 = _socket.socketpair()
+        assert posix.get_inheritable(s1.fileno()) is False
+        assert posix.get_inheritable(s2.fileno()) is False
+        s1.close()
+        s2.close()
+
 
 class AppTestNetlink:
     def setup_class(cls):
@@ -826,6 +854,16 @@
         assert cli.family == socket.AF_INET
 
 
+    def test_accept_non_inheritable(self):
+        import _socket, posix
+        cli = _socket.socket()
+        cli.connect(self.serv.getsockname())
+        fileno, addr = self.serv._accept()
+        assert posix.get_inheritable(fileno) is False
+        posix.close(fileno)
+        cli.close()
+
+
 class AppTestErrno:
     spaceconfig = {'usemodules': ['_socket']}
 
diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20140917)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'MAXGROUPS':      'space.wrap(interp_sre.MAXGROUPS)',
         'compile':        'interp_sre.W_SRE_Pattern',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -13,8 +13,8 @@
 #
 # Constants and exposed functions
 
-from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
+from rpython.rlib.rsre import rsre_core, rsre_char
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
@@ -92,6 +92,10 @@
 #
 # SRE_Pattern class
 
+FLAG_NAMES = ["re.TEMPLATE", "re.IGNORECASE", "re.LOCALE", "re.MULTILINE",
+              "re.DOTALL", "re.UNICODE", "re.VERBOSE", "re.DEBUG",
+              "re.ASCII"]
+
 class W_SRE_Pattern(W_Root):
     _immutable_fields_ = ["code", "flags", "num_groups", "w_groupindex"]
 
@@ -99,7 +103,44 @@
         space = self.space
         raise oefmt(space.w_TypeError, "cannot copy this pattern object")
 
-    def make_ctx(self, w_string, pos=0, endpos=sys.maxint):
+    def repr_w(self):
+        space = self.space
+        u = space.unicode_w(space.repr(self.w_pattern))
+        flag_items = []
+        flags = self.flags
+        if self.is_known_unicode():
+            if ((flags & (rsre_char.SRE_FLAG_LOCALE |
+                          rsre_char.SRE_FLAG_UNICODE |
+                          256))     # rsre_char.SRE_FLAG_ASCII
+                    == rsre_char.SRE_FLAG_UNICODE):
+                flags &= ~rsre_char.SRE_FLAG_UNICODE
+        for i, name in enumerate(FLAG_NAMES):
+            if flags & (1 << i):
+                flags -= (1 << i)
+                flag_items.append(name)
+        if flags != 0:
+            flag_items.append('0x%x' % flags)
+        if len(flag_items) == 0:
+            usep = u''
+            uflags = u''
+        else:
+            usep = u', '
+            uflags = u'|'.join([item.decode('latin-1') for item in flag_items])
+        return space.wrap(u're.compile(%s%s%s)' % (u, usep, uflags))
+
+    def is_known_bytes(self):
+        space = self.space
+        if space.is_none(self.w_pattern):
+            return False
+        return not space.isinstance_w(self.w_pattern, space.w_unicode)
+
+    def is_known_unicode(self):
+        space = self.space
+        if space.is_none(self.w_pattern):
+            return False
+        return space.isinstance_w(self.w_pattern, space.w_unicode)
+
+    def make_ctx(self, w_string, pos=0, endpos=sys.maxint, flags=0):
         """Make a StrMatchContext, BufMatchContext or a UnicodeMatchContext for
         searching in the given w_string object."""
         space = self.space
@@ -107,10 +148,10 @@
             pos = 0
         if endpos < pos:
             endpos = pos
+        flags = self.flags | flags
         if space.isinstance_w(w_string, space.w_unicode):
             unicodestr = space.unicode_w(w_string)
-            if not (space.is_none(self.w_pattern) or
-                    space.isinstance_w(self.w_pattern, space.w_unicode)):
+            if self.is_known_bytes():
                 raise oefmt(space.w_TypeError,
                             "can't use a bytes pattern on a string-like "
                             "object")
@@ -119,10 +160,9 @@
             if endpos > len(unicodestr):
                 endpos = len(unicodestr)
             return rsre_core.UnicodeMatchContext(self.code, unicodestr,
-                                                 pos, endpos, self.flags)
+                                                 pos, endpos, flags)
         elif space.isinstance_w(w_string, space.w_str):
-            if (not space.is_none(self.w_pattern) and
-                space.isinstance_w(self.w_pattern, space.w_unicode)):
+            if self.is_known_unicode():
                 raise oefmt(space.w_TypeError,
                             "can't use a string pattern on a bytes-like "
                             "object")
@@ -132,11 +172,10 @@
             if endpos > len(str):
                 endpos = len(str)
             return rsre_core.StrMatchContext(self.code, str,
-                                             pos, endpos, self.flags)
+                                             pos, endpos, flags)
         else:
             buf = space.readbuf_w(w_string)
-            if (not space.is_none(self.w_pattern) and
-                space.isinstance_w(self.w_pattern, space.w_unicode)):
+            if self.is_known_unicode():
                 raise oefmt(space.w_TypeError,
                             "can't use a string pattern on a bytes-like "
                             "object")
@@ -147,7 +186,7 @@
             if endpos > size:
                 endpos = size
             return rsre_core.BufMatchContext(self.code, buf,
-                                             pos, endpos, self.flags)
+                                             pos, endpos, flags)
 
     def getmatch(self, ctx, found):
         if found:
@@ -161,6 +200,12 @@
         return self.getmatch(ctx, matchcontext(self.space, ctx))
 
     @unwrap_spec(pos=int, endpos=int)
+    def fullmatch_w(self, w_string, pos=0, endpos=sys.maxint):
+        ctx = self.make_ctx(w_string, pos, endpos)
+        ctx.fullmatch_only = True
+        return self.getmatch(ctx, matchcontext(self.space, ctx))
+
+    @unwrap_spec(pos=int, endpos=int)
     def search_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
         return self.getmatch(ctx, searchcontext(self.space, ctx))
@@ -415,10 +460,12 @@
     __new__      = interp2app(SRE_Pattern__new__),
     __copy__     = interp2app(W_SRE_Pattern.cannot_copy_w),
     __deepcopy__ = interp2app(W_SRE_Pattern.cannot_copy_w),
+    __repr__     = interp2app(W_SRE_Pattern.repr_w),
     __weakref__  = make_weakref_descr(W_SRE_Pattern),
     findall      = interp2app(W_SRE_Pattern.findall_w),
     finditer     = interp2app(W_SRE_Pattern.finditer_w),
     match        = interp2app(W_SRE_Pattern.match_w),
+    fullmatch    = interp2app(W_SRE_Pattern.fullmatch_w),
     scanner      = interp2app(W_SRE_Pattern.finditer_w),    # reuse finditer()
     search       = interp2app(W_SRE_Pattern.search_w),
     split        = interp2app(W_SRE_Pattern.split_w),
diff --git a/pypy/module/_sre/test/support_test_app_sre.py b/pypy/module/_sre/test/support_test_app_sre.py
--- a/pypy/module/_sre/test/support_test_app_sre.py
+++ b/pypy/module/_sre/test/support_test_app_sre.py
@@ -1,6 +1,13 @@
 """Support functions for app-level _sre tests."""
 import locale, _sre
-from sre_constants import OPCODES, ATCODES, CHCODES, MAXREPEAT
+from sre_constants import OPCODES as _OPCODES
+from sre_constants import ATCODES as _ATCODES
+from sre_constants import CHCODES as _CHCODES
+from sre_constants import MAXREPEAT
+
+OPCODES = {_opcode.name.lower(): int(_opcode) for _opcode in _OPCODES}
+ATCODES = {_atcode.name.lower(): int(_atcode) for _atcode in _ATCODES}
+CHCODES = {_chcode.name.lower(): int(_chcode) for _chcode in _CHCODES}
 
 def encode_literal(string):
     opcodes = []
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -116,6 +116,22 @@
         import _sre
         raises(TypeError, _sre.compile, {}, 0, [])
 
+    def test_fullmatch(self):
+        import re
+        assert re.compile(r"ab*c").fullmatch("abbcdef") is None
+        assert re.compile(r"ab*c").fullmatch("abbc") is not None
+        assert re.fullmatch(r"ab*c", "abbbcdef") is None
+        assert re.fullmatch(r"ab*c", "abbbc") is not None
+
+    def test_repr(self):
+        import re
+        r = re.compile(r'f(o"\d)', 0)
+        assert repr(r) == (
+            r"""re.compile('f(o"\\d)')""")
+        r = re.compile(r'f(o"\d)', re.IGNORECASE|re.DOTALL|re.VERBOSE)
+        assert repr(r) == (
+            r"""re.compile('f(o"\\d)', re.IGNORECASE|re.DOTALL|re.VERBOSE)""")
+
 
 class AppTestSreMatch:
     spaceconfig = dict(usemodules=('array', ))
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -356,9 +356,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -458,7 +464,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -610,7 +616,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -154,6 +154,7 @@
 
     def test_readValues(self):
         from winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -167,7 +168,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,8 +119,8 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 Py_CLEANUP_SUPPORTED
 """.split()
@@ -651,6 +651,7 @@
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
         ))
+Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
@@ -974,12 +975,14 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
     global py_fatalerror
     py_fatalerror = rffi.llexternal('%s_FatalError' % prefix,
                                     [CONST_STRING], lltype.Void,
                                     compilation_info=eci)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
     add_fork_hook('child', reinit_tls)
 
 def init_function(func):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,9 +1,13 @@
+from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import buffer
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer)
 from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
 
+# PyObject_GetBuffer has been removed, it is defined in abstract.c
+# PyObject_CheckBuffer is also already defined
+
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
 def PyBuffer_IsContiguous(space, view, fortran):
     """Return 1 if the memory defined by the view is C-style (fortran is
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.4.1-alpha0"
+#define PYPY_VERSION_NUM  0x05040100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -16,7 +16,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    raise NotImplementedError
+    raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
 def PyMemoryView_GET_BUFFER(space, w_obj):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
     Py_buffer, mangle_name, pypy_decl)
 from pypy.module.cpyext.typeobjectdefs import (
-    unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
+    unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
@@ -20,8 +20,12 @@
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import widen
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.sys.version import CPYTHON_VERSION
+
+PY3 = CPYTHON_VERSION[0] == 3
 
 # XXX: Also defined in object.h
 Py_LT = 0
@@ -298,11 +302,23 @@
     # Similar to Py_buffer
     _immutable_ = True
 
-    def __init__(self, ptr, size, w_obj):
+    def __init__(self, ptr, size, w_obj, format='B', shape=None,
+                strides=None, ndim=1, itemsize=1, readonly=True):
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
-        self.readonly = True
+        self.format = format
+        if not shape:
+            self.shape = [size]
+        else:
+            self.shape = shape
+        if not strides:
+            self.strides = [1]
+        else:
+            self.strides = strides
+        self.ndim = ndim 
+        self.itemsize = itemsize
+        self.readonly = readonly
 
     def getlength(self):
         return self.size
@@ -313,14 +329,38 @@
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
 
+    def getformat(self):
+        return self.format
+
+    def getshape(self):
+        return self.shape
+
+    def getitemsize(self):
+        return self.itemsize
+
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
-    with lltype.scoped_alloc(Py_buffer) as view:
-        flags = rffi.cast(rffi.INT_real, 0)
-        ret = generic_cpy_call(space, func_target, w_self, view, flags)
-        if rffi.cast(lltype.Signed, ret) == -1:
+    with lltype.scoped_alloc(Py_buffer) as pybuf:
+        _flags = 0
+        if space.len_w(w_args) > 0:
+            _flags = space.int_w(space.listview(w_args)[0])
+        flags = rffi.cast(rffi.INT_real,_flags)
+        size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+        if widen(size) < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        return space.newbuffer(CPyBuffer(view.c_buf, view.c_len, w_self))
+        ptr = pybuf.c_buf
+        size = pybuf.c_len
+        ndim = widen(pybuf.c_ndim)
+        shape =   [pybuf.c_shape[i]   for i in range(ndim)]
+        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_format:
+            format = rffi.charp2str(pybuf.c_format)
+        else:
+            format = 'B'
+        return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
+                            ndim=ndim, shape=shape, strides=strides,
+                            itemsize=pybuf.c_itemsize,
+                            readonly=widen(pybuf.c_readonly)))
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
@@ -542,6 +582,21 @@
                              w_stararg=w_args, w_starstararg=w_kwds)
             return space.call_args(space.get(new_fn, w_self), args)
         api_func = slot_tp_new.api_func
+    elif name == 'tp_as_buffer.c_bf_getbuffer':
+        buff_fn = w_type.getdictvalue(space, '__buffer__')
+        if buff_fn is None:
+            return
+        @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
+                rffi.INT_real, header=None, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def buff_w(space, w_self, pybuf, flags):
+            # XXX this is wrong, needs a test
+            raise oefmt(space.w_NotImplemented, 
+                "calling bf_getbuffer on a builtin type not supported yet")
+            #args = Arguments(space, [w_self],
+            #                 w_stararg=w_args, w_starstararg=w_kwds)
+            #return space.call_args(space.get(buff_fn, w_self), args)
+        api_func = buff_w.api_func
     else:

From pypy.commits at gmail.com  Mon Aug 29 11:18:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 08:18:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: inode()
Message-ID: <57c45228.45c8c20a.e6dc0.8c3d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86705:e69a949f6bcb
Date: 2016-08-29 17:17 +0200
http://bitbucket.org/pypy/pypy/changeset/e69a949f6bcb/

Log:	inode()

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -94,9 +94,10 @@
                     break
             #
             known_type = rposix_scandir.get_known_type(entry)
+            inode = rposix_scandir.get_inode(entry)
         finally:
             self._in_next = False
-        direntry = W_DirEntry(self, name, known_type)
+        direntry = W_DirEntry(self, name, known_type, inode)
         return space.wrap(direntry)
 
 
@@ -122,10 +123,11 @@
 class W_DirEntry(W_Root):
     w_path = None
 
-    def __init__(self, scandir_iterator, name, known_type):
+    def __init__(self, scandir_iterator, name, known_type, inode):
         self.space = scandir_iterator.space
         self.scandir_iterator = scandir_iterator
         self.name = name     # always bytes on Posix
+        self.inode = inode
         self.flags = known_type
         assert known_type == (known_type & 255)
         #
@@ -281,6 +283,9 @@
         st = self.get_stat_or_lstat(follow_symlinks)
         return build_stat_result(self.space, st)
 
+    def descr_inode(self, space):
+        return space.wrap(self.inode)
+
 
 W_DirEntry.typedef = TypeDef(
     'posix.DirEntry',
@@ -294,5 +299,6 @@
     is_file = interp2app(W_DirEntry.descr_is_file),
     is_symlink = interp2app(W_DirEntry.descr_is_symlink),
     stat = interp2app(W_DirEntry.descr_stat),
+    inode = interp2app(W_DirEntry.descr_inode),
 )
 W_DirEntry.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -157,3 +157,10 @@
     def test_fdopendir_unsupported(self):
         posix = self.posix
         raises(TypeError, posix.scandir, 1234)
+
+    def test_inode(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert d.name == 'file1'
+        ino = d.inode()
+        assert ino == d.stat().st_ino
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -623,7 +623,8 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1)),
+             ('d_ino', lltype.Signed)]
             + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
         if HAVE_D_TYPE:
             DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -50,3 +50,6 @@
     if rposix.HAVE_D_TYPE:
         return rffi.getintfield(direntp, 'c_d_type')
     return DT_UNKNOWN
+
+def get_inode(direntp):
+    return rffi.getintfield(direntp, 'c_d_ino')

From pypy.commits at gmail.com  Mon Aug 29 11:21:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 08:21:33 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: "Done"
Message-ID: <57c452fd.44ce1c0a.361ff.b8b5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5690:b3500c9d9877
Date: 2016-08-29 17:21 +0200
http://bitbucket.org/pypy/extradoc/changeset/b3500c9d9877/

Log:	"Done"

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -11,7 +11,8 @@
   We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
   Matti's opinion on that would be great!
 
-* arigo: pathlib: Object-oriented filesystem paths (PEP 428)
+* arigo: look at test failures relaced to os.scandir() or the pathlib
+  module, or the enum module
 
 
 Finished
@@ -75,8 +76,6 @@
   (DONE, maybe a small optimization left---TYPE_*ASCII*---that
   depends on compact unicode representation)
 
-* enum: Support for enumeration types (PEP 435).
-  (PURELY-APPLEVEL, nothing to do, except look at test failures showing
-  other things not implemented like always)
+* enum: Support for enumeration types (PEP 435). (PURELY-APPLEVEL)
 
-* pathlib: Object-oriented filesystem paths (PEP 428).
+* pathlib: Object-oriented filesystem paths (PEP 428). (PURELY-APPLEVEL)

From pypy.commits at gmail.com  Mon Aug 29 11:30:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 08:30:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-scandir: DirEntry.__repr__()
Message-ID: <57c45520.469d1c0a.bcbc7.c715@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-scandir
Changeset: r86706:1f468263bbc2
Date: 2016-08-29 17:29 +0200
http://bitbucket.org/pypy/pypy/changeset/1f468263bbc2/

Log:	DirEntry.__repr__()

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -136,6 +136,10 @@
             w_name = self.space.fsdecode(w_name)
         self.w_name = w_name
 
+    def descr_repr(self, space):
+        u = space.unicode_w(space.repr(self.w_name))
+        return space.wrap(u"<DirEntry %s>" % u)
+
     def fget_name(self, space):
         return self.w_name
 
@@ -289,6 +293,7 @@
 
 W_DirEntry.typedef = TypeDef(
     'posix.DirEntry',
+    __repr__ = interp2app(W_DirEntry.descr_repr),
     name = GetSetProperty(W_DirEntry.fget_name,
                           doc="the entry's base filename, relative to "
                               'scandir() "path" argument'),
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -164,3 +164,8 @@
         assert d.name == 'file1'
         ino = d.inode()
         assert ino == d.stat().st_ino
+
+    def test_repr(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert repr(d) == "<DirEntry 'file1'>"

From pypy.commits at gmail.com  Mon Aug 29 12:37:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 09:37:05 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fixes some calls to functions through
 Windows' COM interface
Message-ID: <57c464b1.4152c20a.199e7.a49c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86707:f4224510b428
Date: 2016-08-29 18:36 +0200
http://bitbucket.org/pypy/pypy/changeset/f4224510b428/

Log:	Fixes some calls to functions through Windows' COM interface

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None

From pypy.commits at gmail.com  Mon Aug 29 12:38:00 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 29 Aug 2016 09:38:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: Move the definition of FakeSpace to the
 top level, instead of nested 4 levels down
Message-ID: <57c464e8.a699c20a.32011.a8dd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86708:7d5495ded17e
Date: 2016-08-29 17:37 +0100
http://bitbucket.org/pypy/pypy/changeset/7d5495ded17e/

Log:	Move the definition of FakeSpace to the top level, instead of nested
	4 levels down

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -145,6 +145,24 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
+class FakeSpace(object):
+    """Like TinyObjSpace, but different"""
+    def __init__(self, config):
+        from distutils.sysconfig import get_python_inc
+        self.config = config
+        self.include_dir = get_python_inc()
+
+    def passthrough(self, arg):
+        return arg
+    listview = passthrough
+    str_w = passthrough
+
+    def unwrap(self, args):
+        try:
+            return args.str_w(None)
+        except:
+            return args
+
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -433,21 +451,8 @@
         self.imported_module_names = []
 
         if self.runappdirect:
+            fake = FakeSpace(self.space.config)
             def interp2app(func):
-                from distutils.sysconfig import get_python_inc
-                class FakeSpace(object):
-                    def passthrough(self, arg):
-                        return arg
-                    listview = passthrough
-                    str_w = passthrough
-                    def unwrap(self, args):
-                        try:
-                            return args.str_w(None)
-                        except:
-                            return args
-                fake = FakeSpace()
-                fake.include_dir = get_python_inc()
-                fake.config = self.space.config
                 def run(*args, **kwargs):
                     for k in kwargs.keys():
                         if k not in func.unwrap_spec and not k.startswith('w_'):

From pypy.commits at gmail.com  Mon Aug 29 14:32:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 11:32:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: Probable fix for issue #2383: have
 'list(S())' call 'S.__getitem__' if S
Message-ID: <57c47fc4.64f9c20a.aba20.84da@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86709:990f5b2322e1
Date: 2016-08-29 20:32 +0200
http://bitbucket.org/pypy/pypy/changeset/990f5b2322e1/

Log:	Probable fix for issue #2383: have 'list(S())' call 'S.__getitem__'
	if S is a subclass of str with a custom __getitem__.

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -58,6 +58,20 @@
     return w_iter
 tuple_iter._annspecialcase_ = 'specialize:memo'
 
+def str_getitem(space):
+    "Utility that returns the app-level descriptor str.__getitem__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_str,
+                                               '__getitem__')
+    return w_iter
+str_getitem._annspecialcase_ = 'specialize:memo'
+
+def unicode_getitem(space):
+    "Utility that returns the app-level descriptor unicode.__getitem__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
+                                               '__getitem__')
+    return w_iter
+unicode_getitem._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     if w_descr is None:
         raise oefmt(space.w_AttributeError,
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -445,7 +445,7 @@
             return w_obj.listview_bytes()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_bytes()
-        if isinstance(w_obj, W_BytesObject) and self._uses_no_iter(w_obj):
+        if isinstance(w_obj, W_BytesObject) and self._str_uses_no_iter(w_obj):
             return w_obj.listview_bytes()
         if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
             return w_obj.getitems_bytes()
@@ -460,7 +460,7 @@
             return w_obj.listview_unicode()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_unicode()
-        if isinstance(w_obj, W_UnicodeObject) and self._uses_no_iter(w_obj):
+        if isinstance(w_obj, W_UnicodeObject) and self._uni_uses_no_iter(w_obj):
             return w_obj.listview_unicode()
         if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
             return w_obj.getitems_unicode()
@@ -504,8 +504,15 @@
         from pypy.objspace.descroperation import tuple_iter
         return self.lookup(w_obj, '__iter__') is tuple_iter(self)
 
-    def _uses_no_iter(self, w_obj):
-        return self.lookup(w_obj, '__iter__') is None
+    def _str_uses_no_iter(self, w_obj):
+        from pypy.objspace.descroperation import str_getitem
+        return (self.lookup(w_obj, '__iter__') is None and
+                self.lookup(w_obj, '__getitem__') is str_getitem(self))
+
+    def _uni_uses_no_iter(self, w_obj):
+        from pypy.objspace.descroperation import unicode_getitem
+        return (self.lookup(w_obj, '__iter__') is None and
+                self.lookup(w_obj, '__getitem__') is unicode_getitem(self))
 
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -432,7 +432,7 @@
 
 
 class AppTestListObject(object):
-    spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
+    #spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
 
     def setup_class(cls):
         import platform
@@ -1518,6 +1518,16 @@
             def __iter__(self):
                 yield "ok"
         assert list(U(u"don't see me")) == ["ok"]
+        #
+        class S(str):
+            def __getitem__(self, index):
+                return str.__getitem__(self, index).upper()
+        assert list(S("abc")) == list("ABC")
+        #
+        class U(unicode):
+            def __getitem__(self, index):
+                return unicode.__getitem__(self, index).upper()
+        assert list(U(u"abc")) == list(u"ABC")
 
     def test_extend_from_nonempty_list_with_subclasses(self):
         l = ["hi!"]
@@ -1543,6 +1553,20 @@
         l.extend(U(u"don't see me"))
         #
         assert l == ["hi!", "okT", "okL", "okL", "okS", "okU"]
+        #
+        class S(str):
+            def __getitem__(self, index):
+                return str.__getitem__(self, index).upper()
+        l = []
+        l.extend(S("abc"))
+        assert l == list("ABC")
+        #
+        class U(unicode):
+            def __getitem__(self, index):
+                return unicode.__getitem__(self, index).upper()
+        l = []
+        l.extend(U(u"abc"))
+        assert l == list(u"ABC")
 
     def test_no_len_on_range_iter(self):
         iterable = range(10)

From pypy.commits at gmail.com  Mon Aug 29 15:02:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 12:02:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c486d1.05371c0a.8370b.0cd4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86710:5e30f5104630
Date: 2016-08-29 21:02 +0200
http://bitbucket.org/pypy/pypy/changeset/5e30f5104630/

Log:	hg merge default

	with suitable refactorings of 990f5b2322e1 for py3k

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -145,6 +145,24 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
+class FakeSpace(object):
+    """Like TinyObjSpace, but different"""
+    def __init__(self, config):
+        from distutils.sysconfig import get_python_inc
+        self.config = config
+        self.include_dir = get_python_inc()
+
+    def passthrough(self, arg):
+        return arg
+    listview = passthrough
+    str_w = passthrough
+
+    def unwrap(self, args):
+        try:
+            return args.str_w(None)
+        except:
+            return args
+
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -440,21 +458,8 @@
         self.imported_module_names = []
 
         if self.runappdirect:
+            fake = FakeSpace(self.space.config)
             def interp2app(func):
-                from distutils.sysconfig import get_python_inc
-                class FakeSpace(object):
-                    def passthrough(self, arg):
-                        return arg
-                    listview = passthrough
-                    str_w = passthrough
-                    def unwrap(self, args):
-                        try:
-                            return args.str_w(None)
-                        except:
-                            return args
-                fake = FakeSpace()
-                fake.include_dir = get_python_inc()
-                fake.config = self.space.config
                 def run(*args, **kwargs):
                     for k in kwargs.keys():
                         if k not in func.unwrap_spec and not k.startswith('w_'):
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -926,7 +926,6 @@
         of the specified width.  B is never truncated.
         """
 
-
 W_BytearrayObject.typedef = TypeDef(
     "bytearray",
     __doc__ = BytearrayDocstrings.__doc__,
diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -114,6 +114,31 @@
         return w_item
 
 
+class W_StringIterObject(W_AbstractSeqIterObject):
+    """Sequence iterator specialized for string-like objects, used
+    for bytes.__iter__() or str.__iter__() or bytearray.__iter__().
+    Needed because otherwise these methods would call the possibly
+    overridden __getitem__() method, which they must not.
+    """
+    def __init__(self, w_seq, getitem_fn):
+        W_AbstractSeqIterObject.__init__(self, w_seq)
+        self.getitem_fn = getitem_fn
+
+    def descr_next(self, space):
+        if self.w_seq is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        index = self.index
+        try:
+            w_item = self.getitem_fn(self.w_seq, space, index)
+        except OperationError as e:
+            self.w_seq = None
+            if not e.match(space, space.w_IndexError):
+                raise
+            raise OperationError(space.w_StopIteration, space.w_None)
+        self.index = index + 1
+        return w_item
+
+
 class W_ReverseSeqIterObject(W_Root):
     def __init__(self, space, w_seq, index=-1):
         self.w_seq = w_seq
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -75,7 +75,8 @@
         return space.wrap(self._len())
 
     def descr_iter(self, space):
-        return space.newseqiter(self)
+        from pypy.objspace.std.iterobject import W_StringIterObject
+        return W_StringIterObject(self, self.__class__._getitem_result)
 
     def descr_contains(self, space, w_sub):
         value = self._val(space)
@@ -133,14 +134,15 @@
         return self._getitem_result(space, index)
 
     def _getitem_result(self, space, index):
+        # Returns the result of 'self[index]', where index is an unwrapped int.
+        # Used by descr_getitem() and by descr_iter().
         selfvalue = self._val(space)
         try:
             character = selfvalue[index]
         except IndexError:
             raise oefmt(space.w_IndexError, "string index out of range")
         from pypy.objspace.std.bytesobject import W_BytesObject
-        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-        if isinstance(self, W_BytesObject) or isinstance(self, W_BytearrayObject):
+        if isinstance(self, W_BytesObject):
             return space.wrap(ord(character))
         return self._new(character)
 
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -432,7 +432,7 @@
 
 
 class AppTestListObject(object):
-    spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
+    #spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
 
     def setup_class(cls):
         import platform
@@ -1524,6 +1524,16 @@
             def __iter__(self):
                 yield "ok"
         assert list(U("don't see me")) == ["ok"]
+        #
+        class S(bytes):
+            def __getitem__(self, index):
+                never_called
+        assert list(S(b"abc")) == list(b"abc")   # __getitem__ ignored
+        #
+        class U(str):
+            def __getitem__(self, index):
+                never_called
+        assert list(U("abc")) == list("abc")     # __getitem__ ignored
 
     def test_extend_from_nonempty_list_with_subclasses(self):
         l = ["hi!"]
@@ -1549,6 +1559,20 @@
         l.extend(U("don't see me"))
         #
         assert l == ["hi!", "okT", "okL", "okL", "okS", "okU"]
+        #
+        class S(bytes):
+            def __getitem__(self, index):
+                never_called
+        l = []
+        l.extend(S(b"abc"))
+        assert l == list(b"abc")    # __getitem__ ignored
+        #
+        class U(str):
+            def __getitem__(self, index):
+                never_called
+        l = []
+        l.extend(U("abc"))
+        assert l == list("abc")     # __getitem__ ignored
 
     def test_issue1266(self):
         l = list(range(1))

From pypy.commits at gmail.com  Mon Aug 29 15:18:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 12:18:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c48a77.4317c20a.8297b.e820@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86711:9aff532b6729
Date: 2016-08-29 21:04 +0200
http://bitbucket.org/pypy/pypy/changeset/9aff532b6729/

Log:	hg merge py3k

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -145,6 +145,24 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
+class FakeSpace(object):
+    """Like TinyObjSpace, but different"""
+    def __init__(self, config):
+        from distutils.sysconfig import get_python_inc
+        self.config = config
+        self.include_dir = get_python_inc()
+
+    def passthrough(self, arg):
+        return arg
+    listview = passthrough
+    str_w = passthrough
+
+    def unwrap(self, args):
+        try:
+            return args.str_w(None)
+        except:
+            return args
+
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -440,21 +458,8 @@
         self.imported_module_names = []
 
         if self.runappdirect:
+            fake = FakeSpace(self.space.config)
             def interp2app(func):
-                from distutils.sysconfig import get_python_inc
-                class FakeSpace(object):
-                    def passthrough(self, arg):
-                        return arg
-                    listview = passthrough
-                    str_w = passthrough
-                    def unwrap(self, args):
-                        try:
-                            return args.str_w(None)
-                        except:
-                            return args
-                fake = FakeSpace()
-                fake.include_dir = get_python_inc()
-                fake.config = self.space.config
                 def run(*args, **kwargs):
                     for k in kwargs.keys():
                         if k not in func.unwrap_spec and not k.startswith('w_'):
diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -114,6 +114,31 @@
         return w_item
 
 
+class W_StringIterObject(W_AbstractSeqIterObject):
+    """Sequence iterator specialized for string-like objects, used
+    for bytes.__iter__() or str.__iter__() or bytearray.__iter__().
+    Needed because otherwise these methods would call the possibly
+    overridden __getitem__() method, which they must not.
+    """
+    def __init__(self, w_seq, getitem_fn):
+        W_AbstractSeqIterObject.__init__(self, w_seq)
+        self.getitem_fn = getitem_fn
+
+    def descr_next(self, space):
+        if self.w_seq is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        index = self.index
+        try:
+            w_item = self.getitem_fn(self.w_seq, space, index)
+        except OperationError as e:
+            self.w_seq = None
+            if not e.match(space, space.w_IndexError):
+                raise
+            raise OperationError(space.w_StopIteration, space.w_None)
+        self.index = index + 1
+        return w_item
+
+
 class W_ReverseSeqIterObject(W_Root):
     def __init__(self, space, w_seq, index=-1):
         self.w_seq = w_seq
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -75,7 +75,8 @@
         return space.wrap(self._len())
 
     def descr_iter(self, space):
-        return space.newseqiter(self)
+        from pypy.objspace.std.iterobject import W_StringIterObject
+        return W_StringIterObject(self, self.__class__._getitem_result)
 
     def descr_contains(self, space, w_sub):
         value = self._val(space)
@@ -133,14 +134,15 @@
         return self._getitem_result(space, index)
 
     def _getitem_result(self, space, index):
+        # Returns the result of 'self[index]', where index is an unwrapped int.
+        # Used by descr_getitem() and by descr_iter().
         selfvalue = self._val(space)
         try:
             character = selfvalue[index]
         except IndexError:
             raise oefmt(space.w_IndexError, "string index out of range")
         from pypy.objspace.std.bytesobject import W_BytesObject
-        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-        if isinstance(self, W_BytesObject) or isinstance(self, W_BytearrayObject):
+        if isinstance(self, W_BytesObject):
             return space.wrap(ord(character))
         return self._new(character)
 
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -432,7 +432,7 @@
 
 
 class AppTestListObject(object):
-    spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
+    #spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
 
     def setup_class(cls):
         import platform
@@ -1524,6 +1524,16 @@
             def __iter__(self):
                 yield "ok"
         assert list(U("don't see me")) == ["ok"]
+        #
+        class S(bytes):
+            def __getitem__(self, index):
+                never_called
+        assert list(S(b"abc")) == list(b"abc")   # __getitem__ ignored
+        #
+        class U(str):
+            def __getitem__(self, index):
+                never_called
+        assert list(U("abc")) == list("abc")     # __getitem__ ignored
 
     def test_extend_from_nonempty_list_with_subclasses(self):
         l = ["hi!"]
@@ -1549,6 +1559,20 @@
         l.extend(U("don't see me"))
         #
         assert l == ["hi!", "okT", "okL", "okL", "okS", "okU"]
+        #
+        class S(bytes):
+            def __getitem__(self, index):
+                never_called
+        l = []
+        l.extend(S(b"abc"))
+        assert l == list(b"abc")    # __getitem__ ignored
+        #
+        class U(str):
+            def __getitem__(self, index):
+                never_called
+        l = []
+        l.extend(U("abc"))
+        assert l == list("abc")     # __getitem__ ignored
 
     def test_issue1266(self):
         l = list(range(1))

From pypy.commits at gmail.com  Mon Aug 29 15:19:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 12:19:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3.5-scandir
Message-ID: <57c48ad3.94a51c0a.bea70.0e3d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86712:9c68dd229e4f
Date: 2016-08-29 21:18 +0200
http://bitbucket.org/pypy/pypy/changeset/9c68dd229e4f/

Log:	hg merge py3.5-scandir

diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -94,9 +94,10 @@
                     break
             #
             known_type = rposix_scandir.get_known_type(entry)
+            inode = rposix_scandir.get_inode(entry)
         finally:
             self._in_next = False
-        direntry = W_DirEntry(self, name, known_type)
+        direntry = W_DirEntry(self, name, known_type, inode)
         return space.wrap(direntry)
 
 
@@ -122,10 +123,11 @@
 class W_DirEntry(W_Root):
     w_path = None
 
-    def __init__(self, scandir_iterator, name, known_type):
+    def __init__(self, scandir_iterator, name, known_type, inode):
         self.space = scandir_iterator.space
         self.scandir_iterator = scandir_iterator
         self.name = name     # always bytes on Posix
+        self.inode = inode
         self.flags = known_type
         assert known_type == (known_type & 255)
         #
@@ -134,6 +136,10 @@
             w_name = self.space.fsdecode(w_name)
         self.w_name = w_name
 
+    def descr_repr(self, space):
+        u = space.unicode_w(space.repr(self.w_name))
+        return space.wrap(u"<DirEntry %s>" % u)
+
     def fget_name(self, space):
         return self.w_name
 
@@ -281,9 +287,13 @@
         st = self.get_stat_or_lstat(follow_symlinks)
         return build_stat_result(self.space, st)
 
+    def descr_inode(self, space):
+        return space.wrap(self.inode)
+
 
 W_DirEntry.typedef = TypeDef(
     'posix.DirEntry',
+    __repr__ = interp2app(W_DirEntry.descr_repr),
     name = GetSetProperty(W_DirEntry.fget_name,
                           doc="the entry's base filename, relative to "
                               'scandir() "path" argument'),
@@ -294,5 +304,6 @@
     is_file = interp2app(W_DirEntry.descr_is_file),
     is_symlink = interp2app(W_DirEntry.descr_is_symlink),
     stat = interp2app(W_DirEntry.descr_stat),
+    inode = interp2app(W_DirEntry.descr_inode),
 )
 W_DirEntry.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -157,3 +157,15 @@
     def test_fdopendir_unsupported(self):
         posix = self.posix
         raises(TypeError, posix.scandir, 1234)
+
+    def test_inode(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert d.name == 'file1'
+        ino = d.inode()
+        assert ino == d.stat().st_ino
+
+    def test_repr(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert repr(d) == "<DirEntry 'file1'>"
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -623,7 +623,8 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1)),
+             ('d_ino', lltype.Signed)]
             + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
         if HAVE_D_TYPE:
             DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -50,3 +50,6 @@
     if rposix.HAVE_D_TYPE:
         return rffi.getintfield(direntp, 'c_d_type')
     return DT_UNKNOWN
+
+def get_inode(direntp):
+    return rffi.getintfield(direntp, 'c_d_ino')

From pypy.commits at gmail.com  Mon Aug 29 15:20:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 12:20:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: transplant rpython changes
Message-ID: <57c48b18.17a71c0a.481c4.129b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86713:508433399921
Date: 2016-08-29 21:20 +0200
http://bitbucket.org/pypy/pypy/changeset/508433399921/

Log:	transplant rpython changes

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -623,7 +623,8 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1)),
+             ('d_ino', lltype.Signed)]
             + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
         if HAVE_D_TYPE:
             DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -50,3 +50,6 @@
     if rposix.HAVE_D_TYPE:
         return rffi.getintfield(direntp, 'c_d_type')
     return DT_UNKNOWN
+
+def get_inode(direntp):
+    return rffi.getintfield(direntp, 'c_d_ino')

From pypy.commits at gmail.com  Mon Aug 29 15:20:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 12:20:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c48b1a.c186c20a.299e7.d36e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86714:04cb05574b89
Date: 2016-08-29 21:20 +0200
http://bitbucket.org/pypy/pypy/changeset/04cb05574b89/

Log:	hg merge default

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -623,7 +623,8 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1)),
+             ('d_ino', lltype.Signed)]
             + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
         if HAVE_D_TYPE:
             DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -50,3 +50,6 @@
     if rposix.HAVE_D_TYPE:
         return rffi.getintfield(direntp, 'c_d_type')
     return DT_UNKNOWN
+
+def get_inode(direntp):
+    return rffi.getintfield(direntp, 'c_d_ino')

From pypy.commits at gmail.com  Mon Aug 29 15:20:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 Aug 2016 12:20:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c48b1b.a719c20a.7a270.ddf9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86715:f7bd6a9082cf
Date: 2016-08-29 21:20 +0200
http://bitbucket.org/pypy/pypy/changeset/f7bd6a9082cf/

Log:	hg merge py3k


From pypy.commits at gmail.com  Mon Aug 29 17:03:36 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 Aug 2016 14:03:36 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: update contributors list,
 missed this for the release
Message-ID: <57c4a328.02c41c0a.7d8c2.407a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86716:b127faf95f86
Date: 2016-08-30 07:01 +1000
http://bitbucket.org/pypy/pypy/changeset/b127faf95f86/

Log:	update contributors list, missed this for the release

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,115 +360,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
-
-  Heinrich-Heine University, Germany 
-  Open End AB (formerly AB Strakt), Sweden
-  merlinux GmbH, Germany 
-  tismerysoft GmbH, Germany 
-  Logilab Paris, France 
-  DFKI GmbH, Germany 
-  Impara, Germany
-  Change Maker, Sweden 
-  University of California Berkeley, USA
-  Google Inc.
-  King's College London
-
-The PyPy Logo as used by http://speed.pypy.org and others was created
-by Samuel Reis and is distributed on terms of Creative Commons Share Alike
-License.
- 
-License for 'lib-python/2.7'
-============================
-
-Except when otherwise stated (look for LICENSE files or copyright/license
-information at the beginning of each file) the files in the 'lib-python/2.7'
-directory are all copyrighted by the Python Software Foundation and licensed
-under the terms that you can find here: https://docs.python.org/2/license.html
-
-License for 'pypy/module/unicodedata/'
-======================================
-
-The following files are from the website of The Unicode Consortium
-at http://www.unicode.org/.  For the terms of use of these files, see
-http://www.unicode.org/terms_of_use.html .  Or they are derived from
-files from the above website, and the same terms of use apply.
-
-    CompositionExclusions-*.txt
-    EastAsianWidth-*.txt
-    LineBreak-*.txt
-    UnicodeData-*.txt
-    UnihanNumeric-*.txt
-
-License for 'dotviewer/font/'
-=============================
-
-Copyright (C) 2008 The Android Open Source Project
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  
-     http://www.apache.org/licenses/LICENSE-2.0
-  
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Detailed license information is contained in the NOTICE file in the
-directory.
-
-
-Licenses and Acknowledgements for Incorporated Software
-=======================================================
-
-This section is an incomplete, but growing list of licenses and
-acknowledgements for third-party software incorporated in the PyPy
-distribution.
-
-License for 'Tcl/Tk'
---------------------
-
-This copy of PyPy contains library code that may, when used, result in
-the Tcl/Tk library to be loaded.  PyPy also includes code that may be
-regarded as being a copy of some parts of the Tcl/Tk header files.
-You may see a copy of the License for Tcl/Tk in the file
-`lib_pypy/_tkinter/license.terms` included here.
-
-License for 'bzip2'
--------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
-
-    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
-
-License for 'openssl'
----------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-openssl library.  You may see a copy of the License for OpenSSL at
-
-    https://www.openssl.org/source/license.html
-
-License for 'gdbm'
-------------------
-
-The gdbm module includes code from gdbm.h, which is distributed under
-the terms of the GPL license version 2 or any later version.  Thus the
-gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
-under the terms of the GPL license as well.
-
-License for 'rpython/rlib/rvmprof/src'
---------------------------------------
-
-The code is based on gperftools. You may see a copy of the License for it at
-
-    https://github.com/gperftools/gperftools/blob/master/COPYING
+  werat
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat

From pypy.commits at gmail.com  Mon Aug 29 17:03:38 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 Aug 2016 14:03:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: update contributors list,
 missed this for the release
Message-ID: <57c4a32a.02c41c0a.7d8c2.407c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86717:52c98aed521e
Date: 2016-08-30 07:01 +1000
http://bitbucket.org/pypy/pypy/changeset/52c98aed521e/

Log:	update contributors list, missed this for the release

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,115 +360,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
-
-  Heinrich-Heine University, Germany 
-  Open End AB (formerly AB Strakt), Sweden
-  merlinux GmbH, Germany 
-  tismerysoft GmbH, Germany 
-  Logilab Paris, France 
-  DFKI GmbH, Germany 
-  Impara, Germany
-  Change Maker, Sweden 
-  University of California Berkeley, USA
-  Google Inc.
-  King's College London
-
-The PyPy Logo as used by http://speed.pypy.org and others was created
-by Samuel Reis and is distributed on terms of Creative Commons Share Alike
-License.
- 
-License for 'lib-python/2.7'
-============================
-
-Except when otherwise stated (look for LICENSE files or copyright/license
-information at the beginning of each file) the files in the 'lib-python/2.7'
-directory are all copyrighted by the Python Software Foundation and licensed
-under the terms that you can find here: https://docs.python.org/2/license.html
-
-License for 'pypy/module/unicodedata/'
-======================================
-
-The following files are from the website of The Unicode Consortium
-at http://www.unicode.org/.  For the terms of use of these files, see
-http://www.unicode.org/terms_of_use.html .  Or they are derived from
-files from the above website, and the same terms of use apply.
-
-    CompositionExclusions-*.txt
-    EastAsianWidth-*.txt
-    LineBreak-*.txt
-    UnicodeData-*.txt
-    UnihanNumeric-*.txt
-
-License for 'dotviewer/font/'
-=============================
-
-Copyright (C) 2008 The Android Open Source Project
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  
-     http://www.apache.org/licenses/LICENSE-2.0
-  
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Detailed license information is contained in the NOTICE file in the
-directory.
-
-
-Licenses and Acknowledgements for Incorporated Software
-=======================================================
-
-This section is an incomplete, but growing list of licenses and
-acknowledgements for third-party software incorporated in the PyPy
-distribution.
-
-License for 'Tcl/Tk'
---------------------
-
-This copy of PyPy contains library code that may, when used, result in
-the Tcl/Tk library to be loaded.  PyPy also includes code that may be
-regarded as being a copy of some parts of the Tcl/Tk header files.
-You may see a copy of the License for Tcl/Tk in the file
-`lib_pypy/_tkinter/license.terms` included here.
-
-License for 'bzip2'
--------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
-
-    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
-
-License for 'openssl'
----------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-openssl library.  You may see a copy of the License for OpenSSL at
-
-    https://www.openssl.org/source/license.html
-
-License for 'gdbm'
-------------------
-
-The gdbm module includes code from gdbm.h, which is distributed under
-the terms of the GPL license version 2 or any later version.  Thus the
-gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
-under the terms of the GPL license as well.
-
-License for 'rpython/rlib/rvmprof/src'
---------------------------------------
-
-The code is based on gperftools. You may see a copy of the License for it at
-
-    https://github.com/gperftools/gperftools/blob/master/COPYING
+  werat
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat

From pypy.commits at gmail.com  Mon Aug 29 17:10:34 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 Aug 2016 14:10:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: finish release note
Message-ID: <57c4a4ca.a717c20a.2f7c9.0dff@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86718:8c6dda3297e5
Date: 2016-08-30 07:09 +1000
http://bitbucket.org/pypy/pypy/changeset/8c6dda3297e5/

Log:	finish release note

diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -3,7 +3,8 @@
 ============
 
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
-This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
 which now supports the "limited API" mode for c-extensions on 
 CPython >=3.2.
@@ -12,9 +13,7 @@
 support to OpenBSD and Dragon Fly BSD
 
 As always, this release fixed many issues and bugs raised by the
-growing community of PyPy users. 
-
-XXXXX MORE ???
+growing community of PyPy users. We strongly recommend updating.
 
 You can download the PyPy2.7 v5.4 release here:
 
@@ -110,8 +109,8 @@
 
   * (RPython) add `rposix_scandir` portably, needed for Python 3.5
 
-  * Support for memoryview attributes (format, itemsize, ...) which also
-    adds support for `PyMemoryView_FromObject`
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
 
 * Bug Fixes
 
@@ -153,10 +152,6 @@
   * Make `hash(-1)` return -2, as CPython does, and fix all the
     ancilary places this matters
 
-  * Issues reported with our previous release were resolved_ after
-    reports from users on our issue tracker at
-    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
-
   * Fix `PyNumber_Check()` to behave more like CPython
 
   * (VMProf) Try hard to not miss any Python-level frame in the
@@ -169,6 +164,10 @@
   * Fix the mapdict cache for subclasses of builtin types that
     provide a dict
 
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like

From pypy.commits at gmail.com  Mon Aug 29 20:01:05 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 29 Aug 2016 17:01:05 -0700 (PDT)
Subject: [pypy-commit] pypy default: Extract .pyd suffix computation
Message-ID: <57c4ccc1.e16ec20a.9378f.aaab@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86719:2cfcf8bcc2e7
Date: 2016-08-30 01:00 +0100
http://bitbucket.org/pypy/pypy/changeset/2cfcf8bcc2e7/

Log:	Extract .pyd suffix computation

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):

From pypy.commits at gmail.com  Mon Aug 29 20:03:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 29 Aug 2016 17:03:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document branch
Message-ID: <57c4cd61.436ec20a.d0586.2d76@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86720:ea17a58fa887
Date: 2016-08-28 22:47 +0100
http://bitbucket.org/pypy/pypy/changeset/ea17a58fa887/

Log:	Document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,5 @@
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
 
-
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.

From pypy.commits at gmail.com  Tue Aug 30 00:08:31 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Mon, 29 Aug 2016 21:08:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: bah
Message-ID: <57c506bf.c62f1c0a.57e03.8ebb@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r86721:7794e4d221e0
Date: 2016-08-29 21:06 -0700
http://bitbucket.org/pypy/pypy/changeset/7794e4d221e0/

Log:	bah

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1164,14 +1164,14 @@
         walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
         with ralloc as pread, walloc as pwrite:
             if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                fdread = c_open_osfhandle(
-                    rffi.cast(rffi.INTPTR_T, pread[0]), 0)
-                fdwrite = c_open_osfhandle(
-                    rffi.cast(rffi.INTPTR_T, pwrite[0]), 1)
+                hread = pread[0]
+                hwrite = pwrite[0]
+                fdread = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hread), 0)
+                fdwrite = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hwrite), 1)
                 if not (fdread == -1 or fdwrite == -1):
                     return (fdread, fdwrite)
-                rwin32.CloseHandle(pread)
-                rwin32.CloseHandle(pwrite)
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
         raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')

From pypy.commits at gmail.com  Tue Aug 30 02:20:28 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 29 Aug 2016 23:20:28 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: updates plan to impl extended
 slicing next
Message-ID: <57c525ac.031dc20a.f5e91.7d63@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5691:b41a888f410e
Date: 2016-08-30 08:20 +0200
http://bitbucket.org/pypy/extradoc/changeset/b41a888f410e/

Log:	updates plan to impl extended slicing next

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -6,10 +6,11 @@
 
 * Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951 (plan_rich)
   Seems to work, but test suite hangs to verify the CPython tests.
-* tuple indexing for memory view (plan_rich)
+* tuple indexing for memory view, !will be merged soon! (plan_rich)
   Comments: Stronly tied to numpy. Hard to implement, because most of the basics are missing (dimensions/strides)
   We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
   Matti's opinion on that would be great!
+* plan_rich: extended slicing for memory view
 
 * arigo: look at test failures relaced to os.scandir() or the pathlib
   module, or the enum module

From pypy.commits at gmail.com  Tue Aug 30 04:13:50 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 01:13:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <57c5403e.d41a1c0a.3b852.0d43@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86723:a803524ac2de
Date: 2016-08-30 10:12 +0200
http://bitbucket.org/pypy/pypy/changeset/a803524ac2de/

Log:	merge default

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,12 +360,15 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
 
   Heinrich-Heine University, Germany 
   Open End AB (formerly AB Strakt), Sweden
@@ -468,15 +476,3 @@
 
     https://github.com/gperftools/gperftools/blob/master/COPYING
 
-License for 'liblzma and 'lzmaffi'
-----------------------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-liblzma library, which was put in the "public domain":
-
-    http://tukaani.org/xz/
-
-The cffi bindings to liblzma (in lib_pypy/_lzma.py) are derived from
-the lzmaffi project which is distributed under a BSD license:
-
-    https://pypi.python.org/pypi/lzmaffi/0.3.0
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -3,7 +3,8 @@
 ============
 
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
-This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
 which now supports the "limited API" mode for c-extensions on 
 CPython >=3.2.
@@ -12,9 +13,7 @@
 support to OpenBSD and Dragon Fly BSD
 
 As always, this release fixed many issues and bugs raised by the
-growing community of PyPy users. 
-
-XXXXX MORE ???
+growing community of PyPy users. We strongly recommend updating.
 
 You can download the PyPy2.7 v5.4 release here:
 
@@ -110,8 +109,8 @@
 
   * (RPython) add `rposix_scandir` portably, needed for Python 3.5
 
-  * Support for memoryview attributes (format, itemsize, ...) which also
-    adds support for `PyMemoryView_FromObject`
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
 
 * Bug Fixes
 
@@ -153,10 +152,6 @@
   * Make `hash(-1)` return -2, as CPython does, and fix all the
     ancilary places this matters
 
-  * Issues reported with our previous release were resolved_ after
-    reports from users on our issue tracker at
-    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
-
   * Fix `PyNumber_Check()` to behave more like CPython
 
   * (VMProf) Try hard to not miss any Python-level frame in the
@@ -169,6 +164,10 @@
   * Fix the mapdict cache for subclasses of builtin types that
     provide a dict
 
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,5 @@
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
 
-
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -430,9 +430,8 @@
 
 def encode_merge_point(log, compressor, values):
     line = []
-    unrolled = unrolling_iterable(values)
     i = 0
-    for value in unrolled:
+    for value in values:
         line.append(value.encode(log,i,compressor))
         i += 1
     return ''.join(line)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1164,14 +1164,14 @@
         walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
         with ralloc as pread, walloc as pwrite:
             if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                fdread = c_open_osfhandle(
-                    rffi.cast(rffi.INTPTR_T, pread[0]), 0)
-                fdwrite = c_open_osfhandle(
-                    rffi.cast(rffi.INTPTR_T, pwrite[0]), 1)
+                hread = pread[0]
+                hwrite = pwrite[0]
+                fdread = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hread), 0)
+                fdwrite = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hwrite), 1)
                 if not (fdread == -1 or fdwrite == -1):
                     return (fdread, fdwrite)
-                rwin32.CloseHandle(pread)
-                rwin32.CloseHandle(pwrite)
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
         raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')

From pypy.commits at gmail.com  Tue Aug 30 04:13:48 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 01:13:48 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove unrolling_iterable,
 causes issues on py3.5-memoryview
Message-ID: <57c5403c.531d1c0a.94f66.d9ae@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r86722:372a9d7304c9
Date: 2016-08-30 10:10 +0200
http://bitbucket.org/pypy/pypy/changeset/372a9d7304c9/

Log:	remove unrolling_iterable, causes issues on py3.5-memoryview

diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -430,9 +430,8 @@
 
 def encode_merge_point(log, compressor, values):
     line = []
-    unrolled = unrolling_iterable(values)
     i = 0
-    for value in unrolled:
+    for value in values:
         line.append(value.encode(log,i,compressor))
         i += 1
     return ''.join(line)

From pypy.commits at gmail.com  Tue Aug 30 04:13:52 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 01:13:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge py3k
Message-ID: <57c54040.43681c0a.928c0.d318@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86724:b820e0d4b544
Date: 2016-08-30 10:12 +0200
http://bitbucket.org/pypy/pypy/changeset/b820e0d4b544/

Log:	merge py3k

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,12 +360,15 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
 
   Heinrich-Heine University, Germany 
   Open End AB (formerly AB Strakt), Sweden
@@ -468,15 +476,3 @@
 
     https://github.com/gperftools/gperftools/blob/master/COPYING
 
-License for 'liblzma and 'lzmaffi'
-----------------------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-liblzma library, which was put in the "public domain":
-
-    http://tukaani.org/xz/
-
-The cffi bindings to liblzma (in lib_pypy/_lzma.py) are derived from
-the lzmaffi project which is distributed under a BSD license:
-
-    https://pypi.python.org/pypi/lzmaffi/0.3.0
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -3,7 +3,8 @@
 ============
 
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
-This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
 which now supports the "limited API" mode for c-extensions on 
 CPython >=3.2.
@@ -12,9 +13,7 @@
 support to OpenBSD and Dragon Fly BSD
 
 As always, this release fixed many issues and bugs raised by the
-growing community of PyPy users. 
-
-XXXXX MORE ???
+growing community of PyPy users. We strongly recommend updating.
 
 You can download the PyPy2.7 v5.4 release here:
 
@@ -110,8 +109,8 @@
 
   * (RPython) add `rposix_scandir` portably, needed for Python 3.5
 
-  * Support for memoryview attributes (format, itemsize, ...) which also
-    adds support for `PyMemoryView_FromObject`
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
 
 * Bug Fixes
 
@@ -153,10 +152,6 @@
   * Make `hash(-1)` return -2, as CPython does, and fix all the
     ancilary places this matters
 
-  * Issues reported with our previous release were resolved_ after
-    reports from users on our issue tracker at
-    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
-
   * Fix `PyNumber_Check()` to behave more like CPython
 
   * (VMProf) Try hard to not miss any Python-level frame in the
@@ -169,6 +164,10 @@
   * Fix the mapdict cache for subclasses of builtin types that
     provide a dict
 
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,5 @@
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
 
-
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -430,9 +430,8 @@
 
 def encode_merge_point(log, compressor, values):
     line = []
-    unrolled = unrolling_iterable(values)
     i = 0
-    for value in unrolled:
+    for value in values:
         line.append(value.encode(log,i,compressor))
         i += 1
     return ''.join(line)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1164,14 +1164,14 @@
         walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
         with ralloc as pread, walloc as pwrite:
             if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                fdread = c_open_osfhandle(
-                    rffi.cast(rffi.INTPTR_T, pread[0]), 0)
-                fdwrite = c_open_osfhandle(
-                    rffi.cast(rffi.INTPTR_T, pwrite[0]), 1)
+                hread = pread[0]
+                hwrite = pwrite[0]
+                fdread = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hread), 0)
+                fdwrite = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hwrite), 1)
                 if not (fdread == -1 or fdwrite == -1):
                     return (fdread, fdwrite)
-                rwin32.CloseHandle(pread)
-                rwin32.CloseHandle(pwrite)
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
         raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')

From pypy.commits at gmail.com  Tue Aug 30 04:13:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 01:13:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: merge py3.5
Message-ID: <57c54042.a6a5c20a.b957f.9756@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86725:d3b1d21c38e2
Date: 2016-08-30 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/d3b1d21c38e2/

Log:	merge py3.5

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -28,3 +28,5 @@
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,12 +360,15 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
 
   Heinrich-Heine University, Germany 
   Open End AB (formerly AB Strakt), Sweden
@@ -468,15 +476,3 @@
 
     https://github.com/gperftools/gperftools/blob/master/COPYING
 
-License for 'liblzma and 'lzmaffi'
-----------------------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-liblzma library, which was put in the "public domain":
-
-    http://tukaani.org/xz/
-
-The cffi bindings to liblzma (in lib_pypy/_lzma.py) are derived from
-the lzmaffi project which is distributed under a BSD license:
-
-    https://pypi.python.org/pypi/lzmaffi/0.3.0
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -3,7 +3,8 @@
 ============
 
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
-This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
 which now supports the "limited API" mode for c-extensions on 
 CPython >=3.2.
@@ -12,9 +13,7 @@
 support to OpenBSD and Dragon Fly BSD
 
 As always, this release fixed many issues and bugs raised by the
-growing community of PyPy users. 
-
-XXXXX MORE ???
+growing community of PyPy users. We strongly recommend updating.
 
 You can download the PyPy2.7 v5.4 release here:
 
@@ -110,8 +109,8 @@
 
   * (RPython) add `rposix_scandir` portably, needed for Python 3.5
 
-  * Support for memoryview attributes (format, itemsize, ...) which also
-    adds support for `PyMemoryView_FromObject`
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
 
 * Bug Fixes
 
@@ -153,10 +152,6 @@
   * Make `hash(-1)` return -2, as CPython does, and fix all the
     ancilary places this matters
 
-  * Issues reported with our previous release were resolved_ after
-    reports from users on our issue tracker at
-    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
-
   * Fix `PyNumber_Check()` to behave more like CPython
 
   * (VMProf) Try hard to not miss any Python-level frame in the
@@ -169,6 +164,10 @@
   * Fix the mapdict cache for subclasses of builtin types that
     provide a dict
 
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,5 @@
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
 
-
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -13,7 +13,7 @@
 #
 # Constants and exposed functions
 
-from rpython.rlib.rsre import rsre_core
+from rpython.rlib.rsre import rsre_core, rsre_char
 from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, MAXGROUPS, getlower, set_unicode_db
 
 
@@ -92,6 +92,10 @@
 #
 # SRE_Pattern class
 
+FLAG_NAMES = ["re.TEMPLATE", "re.IGNORECASE", "re.LOCALE", "re.MULTILINE",
+              "re.DOTALL", "re.UNICODE", "re.VERBOSE", "re.DEBUG",
+              "re.ASCII"]
+
 class W_SRE_Pattern(W_Root):
     _immutable_fields_ = ["code", "flags", "num_groups", "w_groupindex"]
 
@@ -99,7 +103,44 @@
         space = self.space
         raise oefmt(space.w_TypeError, "cannot copy this pattern object")
 
-    def make_ctx(self, w_string, pos=0, endpos=sys.maxint):
+    def repr_w(self):
+        space = self.space
+        u = space.unicode_w(space.repr(self.w_pattern))
+        flag_items = []
+        flags = self.flags
+        if self.is_known_unicode():
+            if ((flags & (rsre_char.SRE_FLAG_LOCALE |
+                          rsre_char.SRE_FLAG_UNICODE |
+                          256))     # rsre_char.SRE_FLAG_ASCII
+                    == rsre_char.SRE_FLAG_UNICODE):
+                flags &= ~rsre_char.SRE_FLAG_UNICODE
+        for i, name in enumerate(FLAG_NAMES):
+            if flags & (1 << i):
+                flags -= (1 << i)
+                flag_items.append(name)
+        if flags != 0:
+            flag_items.append('0x%x' % flags)
+        if len(flag_items) == 0:
+            usep = u''
+            uflags = u''
+        else:
+            usep = u', '
+            uflags = u'|'.join([item.decode('latin-1') for item in flag_items])
+        return space.wrap(u're.compile(%s%s%s)' % (u, usep, uflags))
+
+    def is_known_bytes(self):
+        space = self.space
+        if space.is_none(self.w_pattern):
+            return False
+        return not space.isinstance_w(self.w_pattern, space.w_unicode)
+
+    def is_known_unicode(self):
+        space = self.space
+        if space.is_none(self.w_pattern):
+            return False
+        return space.isinstance_w(self.w_pattern, space.w_unicode)
+
+    def make_ctx(self, w_string, pos=0, endpos=sys.maxint, flags=0):
         """Make a StrMatchContext, BufMatchContext or a UnicodeMatchContext for
         searching in the given w_string object."""
         space = self.space
@@ -107,10 +148,10 @@
             pos = 0
         if endpos < pos:
             endpos = pos
+        flags = self.flags | flags
         if space.isinstance_w(w_string, space.w_unicode):
             unicodestr = space.unicode_w(w_string)
-            if not (space.is_none(self.w_pattern) or
-                    space.isinstance_w(self.w_pattern, space.w_unicode)):
+            if self.is_known_bytes():
                 raise oefmt(space.w_TypeError,
                             "can't use a bytes pattern on a string-like "
                             "object")
@@ -119,10 +160,9 @@
             if endpos > len(unicodestr):
                 endpos = len(unicodestr)
             return rsre_core.UnicodeMatchContext(self.code, unicodestr,
-                                                 pos, endpos, self.flags)
+                                                 pos, endpos, flags)
         elif space.isinstance_w(w_string, space.w_str):
-            if (not space.is_none(self.w_pattern) and
-                space.isinstance_w(self.w_pattern, space.w_unicode)):
+            if self.is_known_unicode():
                 raise oefmt(space.w_TypeError,
                             "can't use a string pattern on a bytes-like "
                             "object")
@@ -132,11 +172,10 @@
             if endpos > len(str):
                 endpos = len(str)
             return rsre_core.StrMatchContext(self.code, str,
-                                             pos, endpos, self.flags)
+                                             pos, endpos, flags)
         else:
             buf = space.readbuf_w(w_string)
-            if (not space.is_none(self.w_pattern) and
-                space.isinstance_w(self.w_pattern, space.w_unicode)):
+            if self.is_known_unicode():
                 raise oefmt(space.w_TypeError,
                             "can't use a string pattern on a bytes-like "
                             "object")
@@ -147,7 +186,7 @@
             if endpos > size:
                 endpos = size
             return rsre_core.BufMatchContext(self.code, buf,
-                                             pos, endpos, self.flags)
+                                             pos, endpos, flags)
 
     def getmatch(self, ctx, found):
         if found:
@@ -161,6 +200,12 @@
         return self.getmatch(ctx, matchcontext(self.space, ctx))
 
     @unwrap_spec(pos=int, endpos=int)
+    def fullmatch_w(self, w_string, pos=0, endpos=sys.maxint):
+        ctx = self.make_ctx(w_string, pos, endpos)
+        ctx.fullmatch_only = True
+        return self.getmatch(ctx, matchcontext(self.space, ctx))
+
+    @unwrap_spec(pos=int, endpos=int)
     def search_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
         return self.getmatch(ctx, searchcontext(self.space, ctx))
@@ -415,10 +460,12 @@
     __new__      = interp2app(SRE_Pattern__new__),
     __copy__     = interp2app(W_SRE_Pattern.cannot_copy_w),
     __deepcopy__ = interp2app(W_SRE_Pattern.cannot_copy_w),
+    __repr__     = interp2app(W_SRE_Pattern.repr_w),
     __weakref__  = make_weakref_descr(W_SRE_Pattern),
     findall      = interp2app(W_SRE_Pattern.findall_w),
     finditer     = interp2app(W_SRE_Pattern.finditer_w),
     match        = interp2app(W_SRE_Pattern.match_w),
+    fullmatch    = interp2app(W_SRE_Pattern.fullmatch_w),
     scanner      = interp2app(W_SRE_Pattern.finditer_w),    # reuse finditer()
     search       = interp2app(W_SRE_Pattern.search_w),
     split        = interp2app(W_SRE_Pattern.split_w),
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -116,6 +116,22 @@
         import _sre
         raises(TypeError, _sre.compile, {}, 0, [])
 
+    def test_fullmatch(self):
+        import re
+        assert re.compile(r"ab*c").fullmatch("abbcdef") is None
+        assert re.compile(r"ab*c").fullmatch("abbc") is not None
+        assert re.fullmatch(r"ab*c", "abbbcdef") is None
+        assert re.fullmatch(r"ab*c", "abbbc") is not None
+
+    def test_repr(self):
+        import re
+        r = re.compile(r'f(o"\d)', 0)
+        assert repr(r) == (
+            r"""re.compile('f(o"\\d)')""")
+        r = re.compile(r'f(o"\d)', re.IGNORECASE|re.DOTALL|re.VERBOSE)
+        assert repr(r) == (
+            r"""re.compile('f(o"\\d)', re.IGNORECASE|re.DOTALL|re.VERBOSE)""")
+
 
 class AppTestSreMatch:
     spaceconfig = dict(usemodules=('array', ))
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):
@@ -145,6 +151,24 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
+class FakeSpace(object):
+    """Like TinyObjSpace, but different"""
+    def __init__(self, config):
+        from distutils.sysconfig import get_python_inc
+        self.config = config
+        self.include_dir = get_python_inc()
+
+    def passthrough(self, arg):
+        return arg
+    listview = passthrough
+    str_w = passthrough
+
+    def unwrap(self, args):
+        try:
+            return args.str_w(None)
+        except:
+            return args
+
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -440,21 +464,8 @@
         self.imported_module_names = []
 
         if self.runappdirect:
+            fake = FakeSpace(self.space.config)
             def interp2app(func):
-                from distutils.sysconfig import get_python_inc
-                class FakeSpace(object):
-                    def passthrough(self, arg):
-                        return arg
-                    listview = passthrough
-                    str_w = passthrough
-                    def unwrap(self, args):
-                        try:
-                            return args.str_w(None)
-                        except:
-                            return args
-                fake = FakeSpace()
-                fake.include_dir = get_python_inc()
-                fake.config = self.space.config
                 def run(*args, **kwargs):
                     for k in kwargs.keys():
                         if k not in func.unwrap_spec and not k.startswith('w_'):
diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -94,9 +94,10 @@
                     break
             #
             known_type = rposix_scandir.get_known_type(entry)
+            inode = rposix_scandir.get_inode(entry)
         finally:
             self._in_next = False
-        direntry = W_DirEntry(self, name, known_type)
+        direntry = W_DirEntry(self, name, known_type, inode)
         return space.wrap(direntry)
 
 
@@ -122,10 +123,11 @@
 class W_DirEntry(W_Root):
     w_path = None
 
-    def __init__(self, scandir_iterator, name, known_type):
+    def __init__(self, scandir_iterator, name, known_type, inode):
         self.space = scandir_iterator.space
         self.scandir_iterator = scandir_iterator
         self.name = name     # always bytes on Posix
+        self.inode = inode
         self.flags = known_type
         assert known_type == (known_type & 255)
         #
@@ -134,6 +136,10 @@
             w_name = self.space.fsdecode(w_name)
         self.w_name = w_name
 
+    def descr_repr(self, space):
+        u = space.unicode_w(space.repr(self.w_name))
+        return space.wrap(u"<DirEntry %s>" % u)
+
     def fget_name(self, space):
         return self.w_name
 
@@ -281,9 +287,13 @@
         st = self.get_stat_or_lstat(follow_symlinks)
         return build_stat_result(self.space, st)
 
+    def descr_inode(self, space):
+        return space.wrap(self.inode)
+
 
 W_DirEntry.typedef = TypeDef(
     'posix.DirEntry',
+    __repr__ = interp2app(W_DirEntry.descr_repr),
     name = GetSetProperty(W_DirEntry.fget_name,
                           doc="the entry's base filename, relative to "
                               'scandir() "path" argument'),
@@ -294,5 +304,6 @@
     is_file = interp2app(W_DirEntry.descr_is_file),
     is_symlink = interp2app(W_DirEntry.descr_is_symlink),
     stat = interp2app(W_DirEntry.descr_stat),
+    inode = interp2app(W_DirEntry.descr_inode),
 )
 W_DirEntry.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -157,3 +157,15 @@
     def test_fdopendir_unsupported(self):
         posix = self.posix
         raises(TypeError, posix.scandir, 1234)
+
+    def test_inode(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert d.name == 'file1'
+        ino = d.inode()
+        assert ino == d.stat().st_ino
+
+    def test_repr(self):
+        posix = self.posix
+        d = next(posix.scandir(self.dir1))
+        assert repr(d) == "<DirEntry 'file1'>"
diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -114,6 +114,31 @@
         return w_item
 
 
+class W_StringIterObject(W_AbstractSeqIterObject):
+    """Sequence iterator specialized for string-like objects, used
+    for bytes.__iter__() or str.__iter__() or bytearray.__iter__().
+    Needed because otherwise these methods would call the possibly
+    overridden __getitem__() method, which they must not.
+    """
+    def __init__(self, w_seq, getitem_fn):
+        W_AbstractSeqIterObject.__init__(self, w_seq)
+        self.getitem_fn = getitem_fn
+
+    def descr_next(self, space):
+        if self.w_seq is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        index = self.index
+        try:
+            w_item = self.getitem_fn(self.w_seq, space, index)
+        except OperationError as e:
+            self.w_seq = None
+            if not e.match(space, space.w_IndexError):
+                raise
+            raise OperationError(space.w_StopIteration, space.w_None)
+        self.index = index + 1
+        return w_item
+
+
 class W_ReverseSeqIterObject(W_Root):
     def __init__(self, space, w_seq, index=-1):
         self.w_seq = w_seq
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -75,7 +75,8 @@
         return space.wrap(self._len())
 
     def descr_iter(self, space):
-        return space.newseqiter(self)
+        from pypy.objspace.std.iterobject import W_StringIterObject
+        return W_StringIterObject(self, self.__class__._getitem_result)
 
     def descr_contains(self, space, w_sub):
         value = self._val(space)
@@ -133,14 +134,15 @@
         return self._getitem_result(space, index)
 
     def _getitem_result(self, space, index):
+        # Returns the result of 'self[index]', where index is an unwrapped int.
+        # Used by descr_getitem() and by descr_iter().
         selfvalue = self._val(space)
         try:
             character = selfvalue[index]
         except IndexError:
             raise oefmt(space.w_IndexError, "string index out of range")
         from pypy.objspace.std.bytesobject import W_BytesObject
-        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-        if isinstance(self, W_BytesObject) or isinstance(self, W_BytearrayObject):
+        if isinstance(self, W_BytesObject):
             return space.wrap(ord(character))
         return self._new(character)
 
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -432,7 +432,7 @@
 
 
 class AppTestListObject(object):
-    spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
+    #spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
 
     def setup_class(cls):
         import platform
@@ -1524,6 +1524,16 @@
             def __iter__(self):
                 yield "ok"
         assert list(U("don't see me")) == ["ok"]
+        #
+        class S(bytes):
+            def __getitem__(self, index):
+                never_called
+        assert list(S(b"abc")) == list(b"abc")   # __getitem__ ignored
+        #
+        class U(str):
+            def __getitem__(self, index):
+                never_called
+        assert list(U("abc")) == list("abc")     # __getitem__ ignored
 
     def test_extend_from_nonempty_list_with_subclasses(self):
         l = ["hi!"]
@@ -1549,6 +1559,20 @@
         l.extend(U("don't see me"))
         #
         assert l == ["hi!", "okT", "okL", "okL", "okS", "okU"]
+        #
+        class S(bytes):
+            def __getitem__(self, index):
+                never_called
+        l = []
+        l.extend(S(b"abc"))
+        assert l == list(b"abc")    # __getitem__ ignored
+        #
+        class U(str):
+            def __getitem__(self, index):
+                never_called
+        l = []
+        l.extend(U("abc"))
+        assert l == list("abc")     # __getitem__ ignored
 
     def test_issue1266(self):
         l = list(range(1))
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -430,9 +430,8 @@
 
 def encode_merge_point(log, compressor, values):
     line = []
-    unrolled = unrolling_iterable(values)
     i = 0
-    for value in unrolled:
+    for value in values:
         line.append(value.encode(log,i,compressor))
         i += 1
     return ''.join(line)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -623,7 +623,8 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1)),
+             ('d_ino', lltype.Signed)]
             + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
         if HAVE_D_TYPE:
             DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
@@ -1159,23 +1160,18 @@
     # 'flags' might be ignored.  Check the result.
     if _WIN32:
         # 'flags' ignored
-        pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        try:
-            ok = CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0)
-            hread = rffi.cast(rffi.INTPTR_T, pread[0])
-            hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
-        finally:
-            lltype.free(pwrite, flavor='raw')
-            lltype.free(pread, flavor='raw')
-        if ok:
-            fdread = c_open_osfhandle(hread, 0)
-            fdwrite = c_open_osfhandle(hwrite, 1)
-            if not (fdread == -1 or fdwrite == -1):
-                return (fdread, fdwrite)
-            rwin32.CloseHandle(pread)
-            rwin32.CloseHandle(pwrite)
+        ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
+        walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
+        with ralloc as pread, walloc as pwrite:
+            if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
+                hread = pread[0]
+                hwrite = pwrite[0]
+                fdread = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hread), 0)
+                fdwrite = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hwrite), 1)
+                if not (fdread == -1 or fdwrite == -1):
+                    return (fdread, fdwrite)
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
         raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -50,3 +50,6 @@
     if rposix.HAVE_D_TYPE:
         return rffi.getintfield(direntp, 'c_d_type')
     return DT_UNKNOWN
+
+def get_inode(direntp):
+    return rffi.getintfield(direntp, 'c_d_ino')
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -89,6 +89,7 @@
     match_end = 0
     match_marks = None
     match_marks_flat = None
+    fullmatch_only = False
 
     def __init__(self, pattern, match_start, end, flags):
         # 'match_start' and 'end' must be known to be non-negative
@@ -526,9 +527,16 @@
         if op == OPCODE_FAILURE:
             return
 
-        if (op == OPCODE_SUCCESS or
-            op == OPCODE_MAX_UNTIL or
-            op == OPCODE_MIN_UNTIL):
+        elif op == OPCODE_SUCCESS:
+            if ctx.fullmatch_only:
+                if ptr != ctx.end:
+                    return     # not a full match
+            ctx.match_end = ptr
+            ctx.match_marks = marks
+            return MATCHED_OK
+
+        elif (op == OPCODE_MAX_UNTIL or
+              op == OPCODE_MIN_UNTIL):
             ctx.match_end = ptr
             ctx.match_marks = marks
             return MATCHED_OK
@@ -551,7 +559,11 @@
             # assert subpattern
             # <ASSERT> <0=skip> <1=back> <pattern>
             ptr1 = ptr - ctx.pat(ppos+1)
-            if ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None:
+            saved = ctx.fullmatch_only
+            ctx.fullmatch_only = False
+            stop = ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None
+            ctx.fullmatch_only = saved
+            if stop:
                 return
             marks = ctx.match_marks
             ppos += ctx.pat(ppos)
@@ -560,7 +572,12 @@
             # assert not subpattern
             # <ASSERT_NOT> <0=skip> <1=back> <pattern>
             ptr1 = ptr - ctx.pat(ppos+1)
-            if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) is not None:
+            saved = ctx.fullmatch_only
+            ctx.fullmatch_only = False
+            stop = (ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks)
+                                      is not None)
+            ctx.fullmatch_only = saved
+            if stop:
                 return
             ppos += ctx.pat(ppos)
 
@@ -999,14 +1016,18 @@
     elif end > length: end = length
     return start, end
 
-def match(pattern, string, start=0, end=sys.maxint, flags=0):
+def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False):
     start, end = _adjust(start, end, len(string))
     ctx = StrMatchContext(pattern, string, start, end, flags)
+    ctx.fullmatch_only = fullmatch
     if match_context(ctx):
         return ctx
     else:
         return None
 
+def fullmatch(pattern, string, start=0, end=sys.maxint, flags=0):
+    return match(pattern, string, start, end, flags, fullmatch=True)
+
 def search(pattern, string, start=0, end=sys.maxint, flags=0):
     start, end = _adjust(start, end, len(string))
     ctx = StrMatchContext(pattern, string, start, end, flags)
diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py
--- a/rpython/rlib/rsre/test/test_match.py
+++ b/rpython/rlib/rsre/test/test_match.py
@@ -272,3 +272,30 @@
         r = get_code("\\{\\{((?:.*?)+)\\}\\}")
         match = rsre_core.match(r, "{{a}}{{b}}")
         assert match.group(1) == "a"
+
+    def test_fullmatch_1(self):
+        r = get_code(r"ab*c")
+        assert not rsre_core.fullmatch(r, "abbbcdef")
+        assert rsre_core.fullmatch(r, "abbbc")
+
+    def test_fullmatch_2(self):
+        r = get_code(r"a(b*?)")
+        match = rsre_core.fullmatch(r, "abbb")
+        assert match.group(1) == "bbb"
+        assert not rsre_core.fullmatch(r, "abbbc")
+
+    def test_fullmatch_3(self):
+        r = get_code(r"a((bp)*?)c")
+        match = rsre_core.fullmatch(r, "abpbpbpc")
+        assert match.group(1) == "bpbpbp"
+
+    def test_fullmatch_4(self):
+        r = get_code(r"a((bp)*)c")
+        match = rsre_core.fullmatch(r, "abpbpbpc")
+        assert match.group(1) == "bpbpbp"
+
+    def test_fullmatch_assertion(self):
+        r = get_code(r"(?=a).b")
+        assert rsre_core.fullmatch(r, "ab")
+        r = get_code(r"(?!a)..")
+        assert not rsre_core.fullmatch(r, "ab")

From pypy.commits at gmail.com  Tue Aug 30 04:47:43 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 01:47:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: missing license entry lost by merge
Message-ID: <57c5482f.482cc20a.cce66.9369@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3k
Changeset: r86726:4ac13ee6aa69
Date: 2016-08-30 10:46 +0200
http://bitbucket.org/pypy/pypy/changeset/4ac13ee6aa69/

Log:	missing license entry lost by merge

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -476,3 +476,15 @@
 
     https://github.com/gperftools/gperftools/blob/master/COPYING
 
+License for 'liblzma and 'lzmaffi'
+----------------------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+liblzma library, which was put in the "public domain":
+
+    http://tukaani.org/xz/
+
+The cffi bindings to liblzma (in lib_pypy/_lzma.py) are derived from
+the lzmaffi project which is distributed under a BSD license:
+
+    https://pypi.python.org/pypi/lzmaffi/0.3.0

From pypy.commits at gmail.com  Tue Aug 30 06:30:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 03:30:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix translation (sorry)
Message-ID: <57c56042.151a1c0a.d0458.0f8c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86727:cc6433b66e6f
Date: 2016-08-30 12:29 +0200
http://bitbucket.org/pypy/pypy/changeset/cc6433b66e6f/

Log:	Fix translation (sorry)

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -441,6 +441,11 @@
     def descr_copy(self, space):
         return self._new(self.data[:])
 
+    @staticmethod
+    def _iter_getitem_result(self, space, index):
+        assert isinstance(self, W_BytearrayObject)
+        return self._getitem_result(space, index)
+
 
 # ____________________________________________________________
 # helpers for slow paths, moved out because they contain loops
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -661,6 +661,11 @@
     def descr_upper(self, space):
         return W_BytesObject(self._value.upper())
 
+    @staticmethod
+    def _iter_getitem_result(self, space, index):
+        assert isinstance(self, W_BytesObject)
+        return self._getitem_result(space, index)
+
 
 def _create_list_from_bytes(value):
     # need this helper function to allow the jit to look inside and inline
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -76,7 +76,7 @@
 
     def descr_iter(self, space):
         from pypy.objspace.std.iterobject import W_StringIterObject
-        return W_StringIterObject(self, self.__class__._getitem_result)
+        return W_StringIterObject(self, self._iter_getitem_result)
 
     def descr_contains(self, space, w_sub):
         value = self._val(space)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -509,6 +509,11 @@
     def _starts_ends_overflow(self, prefix):
         return len(prefix) == 0
 
+    @staticmethod
+    def _iter_getitem_result(self, space, index):
+        assert isinstance(self, W_UnicodeObject)
+        return self._getitem_result(space, index)
+
 
 def _isidentifier(u):
     if not u:

From pypy.commits at gmail.com  Tue Aug 30 07:11:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 04:11:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c569e9.c41f1c0a.c732d.2121@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86728:3808099e7238
Date: 2016-08-30 13:11 +0200
http://bitbucket.org/pypy/pypy/changeset/3808099e7238/

Log:	hg merge py3k

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -476,3 +476,15 @@
 
     https://github.com/gperftools/gperftools/blob/master/COPYING
 
+License for 'liblzma and 'lzmaffi'
+----------------------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+liblzma library, which was put in the "public domain":
+
+    http://tukaani.org/xz/
+
+The cffi bindings to liblzma (in lib_pypy/_lzma.py) are derived from
+the lzmaffi project which is distributed under a BSD license:
+
+    https://pypi.python.org/pypi/lzmaffi/0.3.0
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -446,6 +446,11 @@
     def descr_hex(self, space):
         return _array_to_hexstring(space, self.data, len(self.data), True)
 
+    @staticmethod
+    def _iter_getitem_result(self, space, index):
+        assert isinstance(self, W_BytearrayObject)
+        return self._getitem_result(space, index)
+
 
 # ____________________________________________________________
 # helpers for slow paths, moved out because they contain loops
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -659,6 +659,11 @@
     def descr_upper(self, space):
         return W_BytesObject(self._value.upper())
 
+    @staticmethod
+    def _iter_getitem_result(self, space, index):
+        assert isinstance(self, W_BytesObject)
+        return self._getitem_result(space, index)
+
 
 def _create_list_from_bytes(value):
     # need this helper function to allow the jit to look inside and inline
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -76,7 +76,7 @@
 
     def descr_iter(self, space):
         from pypy.objspace.std.iterobject import W_StringIterObject
-        return W_StringIterObject(self, self.__class__._getitem_result)
+        return W_StringIterObject(self, self._iter_getitem_result)
 
     def descr_contains(self, space, w_sub):
         value = self._val(space)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -509,6 +509,11 @@
     def _starts_ends_overflow(self, prefix):
         return len(prefix) == 0
 
+    @staticmethod
+    def _iter_getitem_result(self, space, index):
+        assert isinstance(self, W_UnicodeObject)
+        return self._getitem_result(space, index)
+
 
 def _isidentifier(u):
     if not u:

From pypy.commits at gmail.com  Tue Aug 30 07:30:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 04:30:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add super.__self__ and
 super.__self_class__
Message-ID: <57c56e72.448e1c0a.28520.28ab@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86729:04632dd29e04
Date: 2016-08-30 13:26 +0200
http://bitbucket.org/pypy/pypy/changeset/04632dd29e04/

Log:	Add super.__self__ and super.__self_class__

diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -85,6 +85,8 @@
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -250,6 +250,17 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 

From pypy.commits at gmail.com  Tue Aug 30 07:31:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 04:31:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: super.__repr__
Message-ID: <57c56e74.68adc20a.41e4b.e3fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86730:93745a358dcc
Date: 2016-08-30 13:30 +0200
http://bitbucket.org/pypy/pypy/changeset/93745a358dcc/

Log:	super.__repr__

diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -23,6 +23,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -84,6 +92,7 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
     __self__         = interp_attrproperty_w("w_self", W_Super),
     __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -261,6 +261,13 @@
         assert super(A, b).__self_class__ is B
         assert super(A).__self_class__ is None
 
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 

From pypy.commits at gmail.com  Tue Aug 30 07:31:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 04:31:02 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <57c56e76.44ce1c0a.bfdef.2c33@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86731:6a614855ec5c
Date: 2016-08-30 13:30 +0200
http://bitbucket.org/pypy/pypy/changeset/6a614855ec5c/

Log:	merge heads

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,115 +360,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
-
-  Heinrich-Heine University, Germany 
-  Open End AB (formerly AB Strakt), Sweden
-  merlinux GmbH, Germany 
-  tismerysoft GmbH, Germany 
-  Logilab Paris, France 
-  DFKI GmbH, Germany 
-  Impara, Germany
-  Change Maker, Sweden 
-  University of California Berkeley, USA
-  Google Inc.
-  King's College London
-
-The PyPy Logo as used by http://speed.pypy.org and others was created
-by Samuel Reis and is distributed on terms of Creative Commons Share Alike
-License.
- 
-License for 'lib-python/2.7'
-============================
-
-Except when otherwise stated (look for LICENSE files or copyright/license
-information at the beginning of each file) the files in the 'lib-python/2.7'
-directory are all copyrighted by the Python Software Foundation and licensed
-under the terms that you can find here: https://docs.python.org/2/license.html
-
-License for 'pypy/module/unicodedata/'
-======================================
-
-The following files are from the website of The Unicode Consortium
-at http://www.unicode.org/.  For the terms of use of these files, see
-http://www.unicode.org/terms_of_use.html .  Or they are derived from
-files from the above website, and the same terms of use apply.
-
-    CompositionExclusions-*.txt
-    EastAsianWidth-*.txt
-    LineBreak-*.txt
-    UnicodeData-*.txt
-    UnihanNumeric-*.txt
-
-License for 'dotviewer/font/'
-=============================
-
-Copyright (C) 2008 The Android Open Source Project
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  
-     http://www.apache.org/licenses/LICENSE-2.0
-  
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Detailed license information is contained in the NOTICE file in the
-directory.
-
-
-Licenses and Acknowledgements for Incorporated Software
-=======================================================
-
-This section is an incomplete, but growing list of licenses and
-acknowledgements for third-party software incorporated in the PyPy
-distribution.
-
-License for 'Tcl/Tk'
---------------------
-
-This copy of PyPy contains library code that may, when used, result in
-the Tcl/Tk library to be loaded.  PyPy also includes code that may be
-regarded as being a copy of some parts of the Tcl/Tk header files.
-You may see a copy of the License for Tcl/Tk in the file
-`lib_pypy/_tkinter/license.terms` included here.
-
-License for 'bzip2'
--------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
-
-    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
-
-License for 'openssl'
----------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-openssl library.  You may see a copy of the License for OpenSSL at
-
-    https://www.openssl.org/source/license.html
-
-License for 'gdbm'
-------------------
-
-The gdbm module includes code from gdbm.h, which is distributed under
-the terms of the GPL license version 2 or any later version.  Thus the
-gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
-under the terms of the GPL license as well.
-
-License for 'rpython/rlib/rvmprof/src'
---------------------------------------
-
-The code is based on gperftools. You may see a copy of the License for it at
-
-    https://github.com/gperftools/gperftools/blob/master/COPYING
+  werat
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -3,7 +3,8 @@
 ============
 
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
-This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
 which now supports the "limited API" mode for c-extensions on 
 CPython >=3.2.
@@ -12,9 +13,7 @@
 support to OpenBSD and Dragon Fly BSD
 
 As always, this release fixed many issues and bugs raised by the
-growing community of PyPy users. 
-
-XXXXX MORE ???
+growing community of PyPy users. We strongly recommend updating.
 
 You can download the PyPy2.7 v5.4 release here:
 
@@ -110,8 +109,8 @@
 
   * (RPython) add `rposix_scandir` portably, needed for Python 3.5
 
-  * Support for memoryview attributes (format, itemsize, ...) which also
-    adds support for `PyMemoryView_FromObject`
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
 
 * Bug Fixes
 
@@ -153,10 +152,6 @@
   * Make `hash(-1)` return -2, as CPython does, and fix all the
     ancilary places this matters
 
-  * Issues reported with our previous release were resolved_ after
-    reports from users on our issue tracker at
-    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
-
   * Fix `PyNumber_Check()` to behave more like CPython
 
   * (VMProf) Try hard to not miss any Python-level frame in the
@@ -169,6 +164,10 @@
   * Fix the mapdict cache for subclasses of builtin types that
     provide a dict
 
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,5 @@
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
 
-
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -430,9 +430,8 @@
 
 def encode_merge_point(log, compressor, values):
     line = []
-    unrolled = unrolling_iterable(values)
     i = 0
-    for value in unrolled:
+    for value in values:
         line.append(value.encode(log,i,compressor))
         i += 1
     return ''.join(line)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1164,14 +1164,14 @@
         walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
         with ralloc as pread, walloc as pwrite:
             if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                fdread = c_open_osfhandle(
-                    rffi.cast(rffi.INTPTR_T, pread[0]), 0)
-                fdwrite = c_open_osfhandle(
-                    rffi.cast(rffi.INTPTR_T, pwrite[0]), 1)
+                hread = pread[0]
+                hwrite = pwrite[0]
+                fdread = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hread), 0)
+                fdwrite = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hwrite), 1)
                 if not (fdread == -1 or fdwrite == -1):
                     return (fdread, fdwrite)
-                rwin32.CloseHandle(pread)
-                rwin32.CloseHandle(pwrite)
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
         raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')

From pypy.commits at gmail.com  Tue Aug 30 08:30:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 05:30:17 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: write down the checksums of 5.4.0
Message-ID: <57c57c59.8aacc20a.f3495.0578@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r780:613e6de57c79
Date: 2016-08-30 14:30 +0200
http://bitbucket.org/pypy/pypy.org/changeset/613e6de57c79/

Log:	write down the checksums of 5.4.0

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -449,31 +449,20 @@
 
 Here are the checksums for each of the downloads
 
-pypy2.7-v5.3 md5::
+pypy2.7-v5.4.0 md5::
 
-    05078bcdd797a025223d5905e2a12332  pypy2-v5.3.0-linux32.tar.bz2
-    7d01e12eaca473258801faebc2db12d8  pypy2-v5.3.0-linux64.tar.bz2
-    bf1640973865c5ca1bc88e299455cbcc  pypy2-v5.3.0-linux-armel.tar.bz2
-    2aa5941a05d46427293d48d67d079df5  pypy2-v5.3.0-linux-armhf-raring.tar.bz2
-    9c2cc832ba15fd4a08ba7e676226f406  pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
-    21a346cca4e8e6897381a0e647a86d68  pypy2-v5.3.0-osx64.tar.bz2
-    c39a578078ab3145d2a584cacf4c164c  pypy2-v5.3.0-s390x.tar.bz2
-    45ce35a438ed8ae1539cc05407d43965  pypy2-v5.3.0-src.tar.bz2
-    24add66f18ab2213c9e44af0ada61085  pypy2-v5.3.0-src.zip
-    f6197adf58bfa32bcb18451289da1c7c  pypy2-v5.3.0-win32.zip
-
-pypy2.7-v5.3.1 md5::
-
-    0ff0e50e9595448d882fe94ab8667993  pypy2-v5.3.1-linux32.tar.bz2
-    41979b51bd5d8f9d6475b6478cf38992  pypy2-v5.3.1-linux64.tar.bz2
-    0f929b98566b154473a50820a3a6cbcf  pypy2-v5.3.1-linux-armel.tar.bz2
-    06ff729d3e30a9787ede69f327534d13  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
-    be714716068e0e0c215e897f7c45ab34  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
-    ae7c15c8b831847f359f673e3461b4e6  pypy2-v5.3.1-osx64.tar.bz2
-    a74f104c25aeb69d2eb2cdce01a2cb02  pypy2-v5.3.1-s390x.tar.bz2
-    2ebc87d24018c60cbf339de039dfecb0  pypy2-v5.3.1-src.tar.bz2
-    a1bbafb57c26de5aae8700cd7667a86b  pypy2-v5.3.1-src.zip
-    38e7d4dd33ea636cc7faebdd94ef6cb4  pypy2-v5.3.1-win32.zip
+    50ea504e66f4d9297f5228d7a3b026ec  pypy2-v5.4.0-linux-armel.tar.bz2
+    e838ba554bc53c793f23c378a898fa0f  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
+    b1b9b755631ef85d400d7690ece50210  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
+    df7180d5070ac19a234fc6c39b88f420  pypy2-v5.4.0-linux32.tar.bz2
+    5e228ba05b6eaa0af37321fd3f425891  pypy2-v5.4.0-linux64.tar.bz2
+    b32d4c97275901665945f1f2813b6f26  pypy2-v5.4.0-osx64.tar.bz2
+    1d32ef8036a9fe718f397813bd070be8  pypy2-v5.4.0-ppc64.tar.bz2
+    d8abb09416b4370ea40c51a710d12b18  pypy2-v5.4.0-ppc64le.tar.bz2
+    b560c2811a3089f22b21db9beea7f273  pypy2-v5.4.0-s390x.tar.bz2
+    c806bea7ecbb999fffeea3a06e6462e8  pypy2-v5.4.0-src.tar.bz2
+    26c2ab1c891651eb620dbde499088c1f  pypy2-v5.4.0-src.zip
+    bd25b15c0d6c0f7c7f6fa75f1da35014  pypy2-v5.4.0-win32.zip
 
 pypy3.3-v5.2-alpha md5::
 
@@ -493,31 +482,20 @@
     009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 
 
-pypy2.7-5.3 sha1::
+pypy2.7-5.4.0 sha1::
 
-    401066f82c8a26dfb1e3421ae4b117655b55ee8d  pypy2-v5.3.0-linux32.tar.bz2
-    939a49319ed8e25ecb9f646ba50f0618eb52c99b  pypy2-v5.3.0-linux64.tar.bz2
-    00fe6e6d672c65d9096b45a708a1be95bca412c4  pypy2-v5.3.0-linux-armel.tar.bz2
-    08fd47ffdf3bc9b7409147e9a0a576e0d577d735  pypy2-v5.3.0-linux-armhf-raring.tar.bz2
-    1572762c5b76a6efda27110267f165bf9b78402d  pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
-    6d6219b7d42b6f31118d2ace2280932dd52d4d9d  pypy2-v5.3.0-osx64.tar.bz2
-    1d4f5f547d798294755fc9e14e2b6490e8a5d194  pypy2-v5.3.0-s390x.tar.bz2
-    7bdb1cfc604192fc2a39023588d648a30975f0e4  pypy2-v5.3.0-src.tar.bz2
-    18a81c46b3d0ecf7e186c19a7d302986a5b15a83  pypy2-v5.3.0-src.zip
-    076251ba3b44435dc11867dab00f392b058bdc7c  pypy2-v5.3.0-win32.zip
-
-pypy2.7-5.3.1 sha1::
-
-    46dcd486ce2acbdf1815b29d70295ad305b667c5  pypy2-v5.3.1-linux32.tar.bz2
-    a06eba349f6346dd6ab4c47f9dcccb0b42986478  pypy2-v5.3.1-linux64.tar.bz2
-    60f33190bae1ac774f461b1da8f974ba1e8c0c24  pypy2-v5.3.1-linux-armel.tar.bz2
-    3fd2fa66437ce72c179ea76522407cde74456e36  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
-    3f7f2aea02e90c6f4a5da00588cd06fdb74aa406  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
-    1d866608f21a465f58ec6f9003f837262f6f7b1a  pypy2-v5.3.1-osx64.tar.bz2
-    08a31fe87ea99864780db86bdb7d7fb9029dc54c  pypy2-v5.3.1-s390x.tar.bz2
-    3abd0c4d133fde7198bf81b15b7786e4e3de9f9f  pypy2-v5.3.1-src.tar.bz2
-    fbfaba410f5031041907a28452cfd5e46b8a7336  pypy2-v5.3.1-src.zip
-    2963d6b1a87f284dfc0336cbec4c37eeb1918f41  pypy2-v5.3.1-win32.zip
+    c50062a83e4bb9fc59b76901c92e7bf1ecd0351f  pypy2-v5.4.0-linux-armel.tar.bz2
+    f4ebad7a9a31dfa55b35cc01b0533ef8e31ab7c4  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
+    c0becdcb7f44e09947afab9df759313ec94563ef  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
+    63be7254bdecd4f3272bcc47f0da7f5db82435a0  pypy2-v5.4.0-linux32.tar.bz2
+    b0e0405ca8f3b143e16122767eb5605d3388af0c  pypy2-v5.4.0-linux64.tar.bz2
+    9c97f54d492886fcaae8611733bcc40a625c8245  pypy2-v5.4.0-osx64.tar.bz2
+    4a263167bbc89447e5adc2ed687ed44798bbca08  pypy2-v5.4.0-ppc64.tar.bz2
+    b3554db74a826fd8e86f1132e9c2cb2e49caac1c  pypy2-v5.4.0-ppc64le.tar.bz2
+    165920a2d0eeda83e8808e7fce93f2a9db7f736a  pypy2-v5.4.0-s390x.tar.bz2
+    95163f8f3c8e9e52e126fc1807d8d94e3d224aec  pypy2-v5.4.0-src.tar.bz2
+    b26546821836cb4bfda0160d37d4dd31fd3aace8  pypy2-v5.4.0-src.zip
+    5ec0ca235cc68b557770b8cf5e1e49bd7b1a0aad  pypy2-v5.4.0-win32.zip
 
 pypy3.3-v5.2-alpha sha1::
 
@@ -531,31 +509,20 @@
     4b31ab492716ea375dd090bbacdf3d7c2d483059  pypy3.3-v5.2.0-alpha1-src.tar.bz2
     d9f5b64f144ebec1a200156809fbbe04fdf7eb7e  pypy3.3-v5.2.0-alpha1-src.zip
 
-pypy2.7-5.3 sha256::
+pypy2.7-5.4.0 sha256::
 
-    bd422fe9d0b7d525d1da3f32855b047bc39ba397d0cf708d8f4f96fe874424f2  pypy2-v5.3.0-linux32.tar.bz2
-    ac336e8877ed676bf87a9a546d5926b6afc4679fa2d3fdf9f3ca56f28ec40588  pypy2-v5.3.0-linux64.tar.bz2
-    81b6f589a947d7353bb69408c46d4833d6e9cb501f3c3f0c73bd28d0e3df69aa  pypy2-v5.3.0-linux-armel.tar.bz2
-    bdb911a87e773a292334061b9c33b907f46d987e403fe94cc627a3b9b1c9cb19  pypy2-v5.3.0-linux-armhf-raring.tar.bz2
-    87b3566b6bbb8bf31c2f0d72bf31d95142fdce004d987812336a59d788005bed  pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
-    1b103bacbdcdbbc490660ec0c7b3d99d1ff1cfc2f13cd403db21c27f03d36a1d  pypy2-v5.3.0-osx64.tar.bz2
-    1ccc0ce08dd55d188d3971331020a1c82e917e418846d2c2c07a225733d85b1e  pypy2-v5.3.0-s390x.tar.bz2
-    4142eb8f403810bc88a4911792bb5a502e152df95806e33e69050c828cd160d5  pypy2-v5.3.0-src.tar.bz2
-    09914006c2c5c394bb6f847f6eb9c9322737c7238e7ca482c5a12c9e2ef76a58  pypy2-v5.3.0-src.zip
-    32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20  pypy2-v5.3.0-win32.zip
-
-pypy2.7-5.3.1 sha256::
-
-    da69f4280b288e524387103eaa3eb4d036965724c3e546da27135c15a77bd2eb  pypy2-v5.3.1-linux32.tar.bz2
-    6d0e8b14875b76b1e77f06a2ee3f1fb5015a645a951ba7a7586289344d4d9c22  pypy2-v5.3.1-linux64.tar.bz2
-    0425f2022c35ef7f0bb3d2b854c5bcbe500b1aba511a0d83581ba6c784913961  pypy2-v5.3.1-linux-armel.tar.bz2
-    b4859496099bde4b17c1e56cc5749dcdcd25b4c68fde1d2ea426de84130e84cc  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
-    5c93eb3c54fbb2c7d7332f775a096671512e590565e6051196bbc5039c5033b5  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
-    7a242d7373b4f18c7f5fe6c2fe6f15e2a405d9adf1f4f934c89b875e60ac5def  pypy2-v5.3.1-osx64.tar.bz2
-    61262f0727ee04b225761b59ce270a64fae9b986d22405a93340f05d0d5c0e0e  pypy2-v5.3.1-s390x.tar.bz2
-    31a52bab584abf3a0f0defd1bf9a29131dab08df43885e7eeddfc7dc9b71836e  pypy2-v5.3.1-src.tar.bz2
-    7eab4a8435583750088001d88371cd0314999b67f26f32ab9d006802f0beec2e  pypy2-v5.3.1-src.zip
-    d83477e2c5f032ebd8c7f47afce03dc8adbeb41a3c74f7db50d9de317dcf3a4a  pypy2-v5.3.1-win32.zip
+    04509044f21bb41ee6d3fafcf637fc0c586c248d4cdae6ac3357606a7b660fdb  pypy2-v5.4.0-linux-armel.tar.bz2
+    95c690bcae6771ebce6cf06c7c2842e0662e007e35162afc963337aa597b471a  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
+    839b08db89b7e20cb670b8cf02596e033ea0b76fb8336af7bedfbb04b6b502da  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
+    ce581270464b14cdecd13dedb9bd7bf98232f767ac4ac282229a405d8e807af1  pypy2-v5.4.0-linux32.tar.bz2
+    bdfea513d59dcd580970cb6f79f3a250d00191fd46b68133d5327e924ca845f8  pypy2-v5.4.0-linux64.tar.bz2
+    3adf21c2bf3432759c99123f21240d71a72aba81d73129e48ef912c34631b723  pypy2-v5.4.0-osx64.tar.bz2
+    dc09a057264dafb7e4bceca57b6a6ba3b0a5273e125a9b29da32b8439f980270  pypy2-v5.4.0-ppc64.tar.bz2
+    4feb0711e7c235b247f8ea0b22e8a676f89e8831488b7a4e9c7f3a6943d07052  pypy2-v5.4.0-ppc64le.tar.bz2
+    6bceb2760b1c7d6105d20207102862160ddddfd9b1a2707b3a8d866ac29e08d3  pypy2-v5.4.0-s390x.tar.bz2
+    d9568ebe9a14d0eaefde887d78f3cba63d665e95c0d234bb583932341f55a655  pypy2-v5.4.0-src.tar.bz2
+    3c165676be8df3b482727438836a9a240ea641392ddd60593f825e1d50029022  pypy2-v5.4.0-src.zip
+    442c0a917781b6155bf78d2648f1ccd9a36c321926a043f83efcea22a99960b4  pypy2-v5.4.0-win32.zip
 
 pypy3.3-v5.2-alpha sha256::
 

From pypy.commits at gmail.com  Tue Aug 30 08:30:56 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 05:30:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: merge py3.5
Message-ID: <57c57c80.05d71c0a.b7e14.4333@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86732:9aeb9147640b
Date: 2016-08-30 13:19 +0200
http://bitbucket.org/pypy/pypy/changeset/9aeb9147640b/

Log:	merge py3.5

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -476,3 +476,15 @@
 
     https://github.com/gperftools/gperftools/blob/master/COPYING
 
+License for 'liblzma and 'lzmaffi'
+----------------------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+liblzma library, which was put in the "public domain":
+
+    http://tukaani.org/xz/
+
+The cffi bindings to liblzma (in lib_pypy/_lzma.py) are derived from
+the lzmaffi project which is distributed under a BSD license:
+
+    https://pypi.python.org/pypi/lzmaffi/0.3.0
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -446,6 +446,11 @@
     def descr_hex(self, space):
         return _array_to_hexstring(space, self.data, len(self.data), True)
 
+    @staticmethod
+    def _iter_getitem_result(self, space, index):
+        assert isinstance(self, W_BytearrayObject)
+        return self._getitem_result(space, index)
+
 
 # ____________________________________________________________
 # helpers for slow paths, moved out because they contain loops
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -663,6 +663,11 @@
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         return _array_to_hexstring(space, StringBuffer(self._value))
 
+    @staticmethod
+    def _iter_getitem_result(self, space, index):
+        assert isinstance(self, W_BytesObject)
+        return self._getitem_result(space, index)
+
 
 def _create_list_from_bytes(value):
     # need this helper function to allow the jit to look inside and inline
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -76,7 +76,7 @@
 
     def descr_iter(self, space):
         from pypy.objspace.std.iterobject import W_StringIterObject
-        return W_StringIterObject(self, self.__class__._getitem_result)
+        return W_StringIterObject(self, self._iter_getitem_result)
 
     def descr_contains(self, space, w_sub):
         value = self._val(space)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -509,6 +509,11 @@
     def _starts_ends_overflow(self, prefix):
         return len(prefix) == 0
 
+    @staticmethod
+    def _iter_getitem_result(self, space, index):
+        assert isinstance(self, W_UnicodeObject)
+        return self._getitem_result(space, index)
+
 
 def _isidentifier(u):
     if not u:

From pypy.commits at gmail.com  Tue Aug 30 08:30:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 05:30:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: extended slicing for
 descr_getitem, part of the test passes already
Message-ID: <57c57c83.4317c20a.8297b.1a4d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86733:fac0fa3c82c0
Date: 2016-08-30 13:35 +0200
http://bitbucket.org/pypy/pypy/changeset/fac0fa3c82c0/

Log:	extended slicing for descr_getitem, part of the test passes already

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -40,6 +40,7 @@
         self.suboffsets = suboffsets
         self.ndim = ndim
         self.flags = 0
+        self.length = -1
         self._init_flags()
 
     # several fields are "overwritten" by the memory view (shape, strides, ...)
@@ -113,6 +114,8 @@
         return buf.getslice(0, n_bytes, 1, n_bytes)
 
     def getlength(self):
+        if self.length != -1:
+            return self.length // self.itemsize
         return self.buf.getlength() // self.itemsize
 
     def descr_tobytes(self, space):
@@ -240,10 +243,40 @@
             buf = SubBuffer(self.buf, start, size)
             return W_MemoryView(buf, self.getformat(), itemsize)
         else:
-            # XXX needs to return a W_MemoryView with a NonContiguousSubBuffer
-            # maybe?  Need to check the cpyext requirements for that
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX extended slicing")
+            mv = W_MemoryView.copy(self)
+            mv.slice(start, stop, step, size)
+            mv.length = mv.bytecount_from_shape()
+            mv._init_flags()
+            return mv
+
+    def slice(self, start, stop, step, size):
+        # modifies the buffer, shape and stride to allow step to be > 1
+        # NOTE that start, stop, are already bytes
+        # TODO subbuffer
+        strides = self.getstrides()[:]
+        shape = self.getshape()[:]
+        itemsize = self.itemsize
+        dim = 0
+        self.buf = SubBuffer(self.buf, start + strides[dim] * (start // itemsize), self.buf.getlength())
+        shape[dim] = size
+        strides[dim] = strides[dim] * step
+        self.strides = strides
+        self.shape = shape
+
+    def bytecount_from_shape(self):
+        dim = self.getndim()
+        shape = self.getshape()
+        length = 1
+        for i in range(dim):
+            length *= shape[i]
+        return length * self.getitemsize()
+
+    @staticmethod
+    def copy(view):
+        # TODO suboffsets
+        return W_MemoryView(view.buf, view.getformat(), view.getitemsize(),
+                            view.getndim(), view.getshape()[:], view.getstrides()[:])
+
 
     def _apply_itemsize(self, space, start, size, itemsize):
         if itemsize > 1:
@@ -466,7 +499,7 @@
         # TODO elif buf.is_contiguous('F'):
         # TODO     flags |= MEMORYVIEW_FORTRAN
 
-        # XXX missing suboffsets
+        # TODO missing suboffsets
 
         self.flags = flags
 

From pypy.commits at gmail.com  Tue Aug 30 08:31:01 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 05:31:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: memoryview extended slicing
 passes first test
Message-ID: <57c57c85.c19d1c0a.11b09.47a8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86734:66ffe66cfca3
Date: 2016-08-30 14:30 +0200
http://bitbucket.org/pypy/pypy/changeset/66ffe66cfca3/

Log:	memoryview extended slicing passes first test

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -251,7 +251,7 @@
 
     def slice(self, start, stop, step, size):
         # modifies the buffer, shape and stride to allow step to be > 1
-        # NOTE that start, stop, are already bytes
+        # NOTE that start, stop, are already byte offsets
         # TODO subbuffer
         strides = self.getstrides()[:]
         shape = self.getshape()[:]
@@ -320,8 +320,29 @@
                             "cannot modify size of memoryview object")
             self.buf.setslice(start, value.as_str())
         else:
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX extended slicing")
+            if self.getndim() != 1:
+                raise oefmt(space.w_NotImplementedError,
+                        "memoryview slice assignments are currently "
+                        "restricted to ndim = 1")
+            itemsize = self.getitemsize()
+            data = []
+            src = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+            dst_strides = self.getstrides()
+            dim = 0
+            dst = SubBuffer(self.buf, start + dst_strides[dim] * (start // itemsize), self.buf.getlength())
+            src_stride0 = dst_strides[dim]
+
+            off = 0
+            src_shape0 = size
+            src_stride0 = src.getstrides()[0]
+            for i in range(src_shape0):
+                data.append(src.getslice(off,off+itemsize,1,itemsize))
+                off += src_stride0
+            off = 0
+            dst_stride0 = self.getstrides()[0] * step
+            for dataslice in data:
+                dst.setslice(off, dataslice)
+                off += dst_stride0
 
     def descr_len(self, space):
         self._check_released(space)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -46,7 +46,7 @@
         assert len(w) == 1
         assert list(w) == [97]
         v[::2] = b'ABC'
-        assert data == bytearray(b'AbBeCg')
+        assert data == bytearray(eval("b'AbBeCg'"))
 
     def test_memoryview_attrs(self):
         v = memoryview(b"a"*100)

From pypy.commits at gmail.com  Tue Aug 30 09:22:23 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 06:22:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: pass another test for extended
 slicing by implementing copying for multi dim. buffer/memoryviews
Message-ID: <57c5888f.10a81c0a.da4cd.576e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86735:0b1786c769bb
Date: 2016-08-30 15:21 +0200
http://bitbucket.org/pypy/pypy/changeset/0b1786c769bb/

Log:	pass another test for extended slicing by implementing copying for
	multi dim. buffer/memoryviews

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -109,9 +109,35 @@
     descr_ne = _make_descr__cmp('ne')
 
     def as_str(self):
+        return ''.join(self.copy_buffer())
+
+    def copy_buffer(self):
         buf = self.buf
         n_bytes = buf.getlength()
-        return buf.getslice(0, n_bytes, 1, n_bytes)
+        data = []
+        self._copy_rec(0, data, 0)
+        return data
+
+    def _copy_rec(self, idim, data, off):
+        shapes = self.getshape()
+        shape = shapes[idim]
+
+        if self.getndim()-1 == idim:
+            self._copy_base(data,off)
+            return
+
+        for i in range(shape):
+            self._copy_rec(idim+1,data,off)
+            off += strides[idim]
+
+    def _copy_base(self, data, off):
+        shapes = self.getshape()
+        step = shapes[0] // self.getitemsize()
+        strides = self.getstrides()
+        for i in range(step):
+            bytes = self.buf.getslice(off, off+self.itemsize, 1, self.itemsize)
+            data.append(bytes)
+            off += strides[0]
 
     def getlength(self):
         if self.length != -1:
@@ -251,15 +277,16 @@
 
     def slice(self, start, stop, step, size):
         # modifies the buffer, shape and stride to allow step to be > 1
-        # NOTE that start, stop, are already byte offsets
+        # NOTE that start & stop are already byte offsets
         # TODO subbuffer
         strides = self.getstrides()[:]
         shape = self.getshape()[:]
         itemsize = self.itemsize
         dim = 0
-        self.buf = SubBuffer(self.buf, start + strides[dim] * (start // itemsize), self.buf.getlength())
+        length = self.buf.getlength()
+        self.buf = SubBuffer(self.buf, strides[dim] * start, size)
         shape[dim] = size
-        strides[dim] = strides[dim] * step
+        strides[dim] = strides[dim] * step * itemsize
         self.strides = strides
         self.shape = shape
 

From pypy.commits at gmail.com  Tue Aug 30 09:33:29 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 30 Aug 2016 06:33:29 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Fix wrong initialization order
Message-ID: <57c58b29.81a2c20a.cf2ff.2647@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86736:d2ff4cbaeae8
Date: 2016-08-19 21:50 +0200
http://bitbucket.org/pypy/pypy/changeset/d2ff4cbaeae8/

Log:	Fix wrong initialization order

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -23,10 +23,11 @@
             ('hash', lltype.Signed))
     #HDR = rffi.COpaque('object_t')
 
-    def malloc_fixedsize_clear(self, typeid16, size,
+    def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False,
                                is_finalizer_light=False,
                                contains_weakptr=False):
+        #return llop.qcgc_allocate(size, typeid)
         raise NotImplementedError
         ## XXX finalizers are ignored for now
         ##ll_assert(not needs_finalizer, 'XXX needs_finalizer')
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -564,6 +564,9 @@
 
     # __________ instrumentation _________
     'instrument_count':     LLOp(),
+
+    # __________ QCGC operations _________
+    'qcgc_allocate':   LLOp(canmallocgc=True),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -472,10 +472,10 @@
         return eci
 
     def gc_startup_code(self):
+        yield 'qcgc_initialize();'
         s = list(super(QcgcFrameworkGcPolicy, self).gc_startup_code())
         for i in s:
             yield i
-        yield 'qcgc_initialize();'
 
 name_to_gcpolicy = {
     'boehm': BoehmGcPolicy,

From pypy.commits at gmail.com  Tue Aug 30 09:33:30 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 30 Aug 2016 06:33:30 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Remove superfluous(?) code
Message-ID: <57c58b2a.c75dc20a.8a59a.2963@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86737:c3c59faf9e4a
Date: 2016-08-19 22:09 +0200
http://bitbucket.org/pypy/pypy/changeset/c3c59faf9e4a/

Log:	Remove superfluous(?) code

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -564,9 +564,6 @@
 
     # __________ instrumentation _________
     'instrument_count':     LLOp(),
-
-    # __________ QCGC operations _________
-    'qcgc_allocate':   LLOp(canmallocgc=True),
 }
 # ***** Run test_lloperation after changes. *****
 

From pypy.commits at gmail.com  Tue Aug 30 09:33:32 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 30 Aug 2016 06:33:32 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Implement write barrier and try
 fix pypy compile error
Message-ID: <57c58b2c.8cc51c0a.bb335.6462@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86738:022badc3be64
Date: 2016-08-30 15:21 +0200
http://bitbucket.org/pypy/pypy/changeset/022badc3be64/

Log:	Implement write barrier and try fix pypy compile error

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -1,5 +1,5 @@
 from rpython.memory.gc.base import GCBase
-from rpython.memory.support import mangle_hash
+#from rpython.memory.support import mangle_hash
 from rpython.rtyper.lltypesystem import rffi, lltype, llgroup, llmemory, llarena
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert
@@ -88,6 +88,14 @@
         #llop.gc_writebarrier(dest_addr)
         #return True
 
+    # XXX: WRITE BARRIER
+    def write_barrier(self, addr_struct):
+        llop.qcgc_write_barrier(lltype.Void, addr_struct)
+
+    def register_finalizer(self, fq_index, gcobj):
+        # XXX: Not supported
+        pass
+
     def id_or_identityhash(self, gcobj, is_hash):
         hdr = self.header(llmemory.cast_ptr_to_adr(gcobj))
         has_hash = (hdr.flags & QCGC_HAS_HASH)
@@ -96,7 +104,7 @@
         if is_hash:
             if has_hash:
                 return i # Do not mangle for objects with built in hash
-            i = mangle_hash(i)
+            i = i ^ (i >> 5)
         return i
 
     def id(self, gcobje):
diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -58,6 +58,25 @@
         for _ in livevars: # Does not move, so no writing back
             hop.genop("qcgc_pop_root", [])
 
+    def gct_gc_fq_register(self, hop):
+        pass
+#        index = self.get_finalizer_queue_index(hop)
+#        c_index = rmodel.inputconst(lltype.Signed, index)
+#        v_ptr = hop.spaceop.args[1]
+#        v_ptr = hop.genop("cast_opaque_ptr", [v_ptr],
+#                          resulttype=llmemory.GCREF)
+#        hop.genop("direct_call", [self.register_finalizer_ptr, self.c_const_gc,
+#                                  c_index, v_ptr])
+
+    def gct_gc_fq_next_dead(self, hop):
+        pass
+#        index = self.get_finalizer_queue_index(hop)
+#        c_ll_next_dead = self.finalizer_handlers[index][2]
+#        v_adr = hop.genop("direct_call", [c_ll_next_dead],
+#                          resulttype=llmemory.Address)
+#        hop.genop("cast_adr_to_ptr", [v_adr],
+#                  resultvar = hop.spaceop.result)
+
 class QcgcRootWalker(BaseRootWalker):
     def walk_stack_roots(self, collect_stack_root, is_minor=False):
         raise NotImplementedError
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -518,6 +518,7 @@
     'qcgc_allocate':    LLOp(canmallocgc=True),
     'qcgc_collect':     LLOp(canmallocgc=True),
     'qcgc_is_prebuilt': LLOp(),
+    'qcgc_write_barrier':   LLOp(),
 
     # __________ weakrefs __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -965,3 +965,7 @@
         result = self.expr(op.result)
         return '%s = (((object_t *) %s)->flags & QCGC_PREBUILT_OBJECT) != 0;' % (
                 result, obj)
+
+    def OP_QCGC_WRITE_BARRIER(self, op):
+        obj = self.expr(op.args[0])
+        return 'qcgc_write(%s);' % (obj,)

From pypy.commits at gmail.com  Tue Aug 30 09:33:34 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 30 Aug 2016 06:33:34 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Merge heads
Message-ID: <57c58b2e.88cb1c0a.55ab1.5c85@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86739:acfdf9dfd37b
Date: 2016-08-30 15:32 +0200
http://bitbucket.org/pypy/pypy/changeset/acfdf9dfd37b/

Log:	Merge heads


From pypy.commits at gmail.com  Tue Aug 30 09:40:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 06:40:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Refactor a bit code for subx(). Add a
 shortcut like CPython's for
Message-ID: <57c58cd0.898b1c0a.85995.6371@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86740:004521738f2e
Date: 2016-08-30 15:39 +0200
http://bitbucket.org/pypy/pypy/changeset/004521738f2e/

Log:	Refactor a bit code for subx(). Add a shortcut like CPython's for
	buffers for the 2nd argument (e.g. bytearray). Fix the general case
	which CPython also gets wrong.

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -140,53 +140,60 @@
             return False
         return space.isinstance_w(self.w_pattern, space.w_unicode)
 
+    def getstring(self, w_string):
+        """Accepts a string-like object (str, bytes, bytearray, buffer...)
+        and returns a tuple (len, rpython_unicode, rpython_str, rpython_buf),
+        where only one of the rpython_xxx is non-None.
+        """
+        unicodestr = None
+        string = None
+        buf = None
+        space = self.space
+        if space.isinstance_w(w_string, space.w_unicode):
+            unicodestr = space.unicode_w(w_string)
+            length = len(unicodestr)
+        elif space.isinstance_w(w_string, space.w_str):
+            string = space.str_w(w_string)
+            length = len(string)
+        else:
+            buf = space.readbuf_w(w_string)
+            length = buf.getlength()
+            assert length >= 0
+        return (length, unicodestr, string, buf)
+
     def make_ctx(self, w_string, pos=0, endpos=sys.maxint, flags=0):
         """Make a StrMatchContext, BufMatchContext or a UnicodeMatchContext for
         searching in the given w_string object."""
         space = self.space
+        length, unicodestr, string, buf = self.getstring(w_string)
         if pos < 0:
             pos = 0
+        elif pos > length:
+            pos = length
         if endpos < pos:
             endpos = pos
+        elif endpos > length:
+            endpos = length
         flags = self.flags | flags
-        if space.isinstance_w(w_string, space.w_unicode):
-            unicodestr = space.unicode_w(w_string)
+        #
+        if unicodestr is not None:
             if self.is_known_bytes():
                 raise oefmt(space.w_TypeError,
                             "can't use a bytes pattern on a string-like "
                             "object")
-            if pos > len(unicodestr):
-                pos = len(unicodestr)
-            if endpos > len(unicodestr):
-                endpos = len(unicodestr)
             return rsre_core.UnicodeMatchContext(self.code, unicodestr,
                                                  pos, endpos, flags)
-        elif space.isinstance_w(w_string, space.w_str):
+        else:
             if self.is_known_unicode():
                 raise oefmt(space.w_TypeError,
                             "can't use a string pattern on a bytes-like "
                             "object")
-            str = space.str_w(w_string)
-            if pos > len(str):
-                pos = len(str)
-            if endpos > len(str):
-                endpos = len(str)
-            return rsre_core.StrMatchContext(self.code, str,
-                                             pos, endpos, flags)
-        else:
-            buf = space.readbuf_w(w_string)
-            if self.is_known_unicode():
-                raise oefmt(space.w_TypeError,
-                            "can't use a string pattern on a bytes-like "
-                            "object")
-            size = buf.getlength()
-            assert size >= 0
-            if pos > size:
-                pos = size
-            if endpos > size:
-                endpos = size
-            return rsre_core.BufMatchContext(self.code, buf,
-                                             pos, endpos, flags)
+            if string is not None:
+                return rsre_core.StrMatchContext(self.code, string,
+                                                 pos, endpos, flags)
+            else:
+                return rsre_core.BufMatchContext(self.code, buf,
+                                                 pos, endpos, flags)
 
     def getmatch(self, ctx, found):
         if found:
@@ -295,27 +302,27 @@
             w_filter = w_ptemplate
             filter_is_callable = True
         else:
-            if space.isinstance_w(w_ptemplate, space.w_unicode):
-                filter_as_unicode = space.unicode_w(w_ptemplate)
+            length, filter_as_unicode, filter_as_string, buf = (
+                self.getstring(w_ptemplate))
+            if filter_as_unicode is not None:
                 literal = u'\\' not in filter_as_unicode
                 use_builder = (
                     space.isinstance_w(w_string, space.w_unicode) and literal)
             else:
-                try:
-                    filter_as_string = space.bytes_w(w_ptemplate)
-                except OperationError as e:
-                    if e.async(space):
-                        raise
-                    literal = False
-                else:
-                    literal = '\\' not in filter_as_string
-                    use_builder = (
-                        space.isinstance_w(w_string, space.w_str) and literal)
+                if buf is not None:
+                    filter_as_string = buf.as_str()
+                literal = '\\' not in filter_as_string
+                use_builder = (
+                    space.isinstance_w(w_string, space.w_str) and literal)
             if literal:
                 w_filter = w_ptemplate
                 filter_is_callable = False
             else:
                 # not a literal; hand it over to the template compiler
+                # FIX for a CPython 3.5 bug: if w_ptemplate is a buffer
+                # (e.g. a bytearray), convert it to a byte string here.
+                if buf is not None:
+                    w_ptemplate = space.newbytes(filter_as_string)
                 w_re = import_re(space)
                 w_filter = space.call_method(w_re, '_subx',
                                              space.wrap(self), w_ptemplate)
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -285,6 +285,12 @@
         import re
         assert re.sub('=\w{2}', 'x', '=CA') == 'x'
 
+    def test_sub_bytearray(self):
+        import re
+        assert re.sub(b'a', bytearray(b'A'), b'axa') == b'AxA'
+        # this fails on CPython 3.5:
+        assert re.sub(b'a', bytearray(b'\\n'), b'axa') == b'\nx\n'
+
     def test_match_array(self):
         import re, array
         a = array.array('b', b'hello')

From pypy.commits at gmail.com  Tue Aug 30 09:46:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 06:46:15 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: updates
Message-ID: <57c58e27.11051c0a.a4c42.61ef@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5692:617eb3b3e6b3
Date: 2016-08-30 15:46 +0200
http://bitbucket.org/pypy/extradoc/changeset/617eb3b3e6b3/

Log:	updates

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -15,6 +15,8 @@
 * arigo: look at test failures relaced to os.scandir() or the pathlib
   module, or the enum module
 
+* arigo: look at and fix the .hex() methods
+
 
 Finished
 --------
diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -37,3 +37,6 @@
   But if we call with a read-only buffer, mutate_flag is ignored (instead
   of rejecting a True value)---ioctl(x, y, "foo", True) will not actually
   mutate the string "foo", but the True is completely ignored.
+
+* re.sub(b'y', bytearray(b'a'), bytearray(b'xyz')) -> b'xaz'
+  re.sub(b'y', bytearray(b'\\n'), bytearray(b'xyz')) -> internal TypeError

From pypy.commits at gmail.com  Tue Aug 30 09:49:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 06:49:03 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: stop looking at hex(),
 py3.5-memoryview has improvements already
Message-ID: <57c58ecf.4152c20a.199e7.2c68@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5693:9197e46895a1
Date: 2016-08-30 15:48 +0200
http://bitbucket.org/pypy/extradoc/changeset/9197e46895a1/

Log:	stop looking at hex(), py3.5-memoryview has improvements already

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -15,8 +15,6 @@
 * arigo: look at test failures relaced to os.scandir() or the pathlib
   module, or the enum module
 
-* arigo: look at and fix the .hex() methods
-
 
 Finished
 --------

From pypy.commits at gmail.com  Tue Aug 30 09:57:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 06:57:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add a pickle test, passes
Message-ID: <57c590c1.d8011c0a.777e7.5f2b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86741:f36adc8b3688
Date: 2016-08-30 15:56 +0200
http://bitbucket.org/pypy/pypy/changeset/f36adc8b3688/

Log:	Add a pickle test, passes

diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -82,6 +82,12 @@
     def teardown_class(cls):
         _detach_helpers(cls.space)
 
+    def test_pickle_basic(self):
+        import pickle
+        pckl = pickle.dumps((u'abc', 0))
+        result = pickle.loads(pckl)
+        assert result == (u'abc', 0)
+
     def test_pickle_code(self):
         def f():
             return 42

From pypy.commits at gmail.com  Tue Aug 30 10:19:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 07:19:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: SRE_Match.__repr__
Message-ID: <57c595ee.a710c20a.54e5.3dad@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86742:897a49107fda
Date: 2016-08-30 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/897a49107fda/

Log:	SRE_Match.__repr__

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -106,6 +106,8 @@
     def repr_w(self):
         space = self.space
         u = space.unicode_w(space.repr(self.w_pattern))
+        if len(u) > 200:
+            u = u[:200]
         flag_items = []
         flags = self.flags
         if self.is_known_unicode():
@@ -497,6 +499,17 @@
         self.srepat = srepat
         self.ctx = ctx
 
+    def repr_w(self):
+        space = self.space
+        ctx = self.ctx
+        start, end = ctx.match_start, ctx.match_end
+        w_s = slice_w(space, ctx, start, end, space.w_None)
+        u = space.unicode_w(space.repr(w_s))
+        if len(u) > 50:
+            u = u[:50]
+        return space.wrap(u'<_sre.SRE_Match object; span=(%d, %d), match=%s>' % 
+                          (start, end, u))
+
     def cannot_copy_w(self):
         space = self.space
         raise oefmt(space.w_TypeError, "cannot copy this match object")
@@ -651,6 +664,7 @@
     'SRE_Match',
     __copy__     = interp2app(W_SRE_Match.cannot_copy_w),
     __deepcopy__ = interp2app(W_SRE_Match.cannot_copy_w),
+    __repr__     = interp2app(W_SRE_Match.repr_w),
     group        = interp2app(W_SRE_Match.group_w),
     groups       = interp2app(W_SRE_Match.groups_w),
     groupdict    = interp2app(W_SRE_Match.groupdict_w),
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -301,6 +301,11 @@
         import re
         raises(TypeError, re.match, 'hel+', list('hello'))
 
+    def test_match_repr(self):
+        import re
+        m = re.search("ab+c", "xabbbcd")
+        assert repr(m) == "<_sre.SRE_Match object; span=(1, 6), match='abbbc'>"
+
     def test_group_bugs(self):
         import re
         r = re.compile(r"""

From pypy.commits at gmail.com  Tue Aug 30 10:28:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 07:28:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: groupindex() returns a read-only
 dictproxy now
Message-ID: <57c597fc.c15e1c0a.42ba2.7495@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86743:c187e8cdb3f6
Date: 2016-08-30 16:27 +0200
http://bitbucket.org/pypy/pypy/changeset/c187e8cdb3f6/

Log:	groupindex() returns a read-only dictproxy now

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -130,6 +130,12 @@
             uflags = u'|'.join([item.decode('latin-1') for item in flag_items])
         return space.wrap(u're.compile(%s%s%s)' % (u, usep, uflags))
 
+    def fget_groupindex(self, space):
+        w_groupindex = self.w_groupindex
+        if space.isinstance_w(w_groupindex, space.w_dict):
+            w_groupindex = space.newdictproxy(w_groupindex)
+        return w_groupindex
+
     def is_known_bytes(self):
         space = self.space
         if space.is_none(self.w_pattern):
@@ -481,7 +487,7 @@
     sub          = interp2app(W_SRE_Pattern.sub_w),
     subn         = interp2app(W_SRE_Pattern.subn_w),
     flags        = interp_attrproperty('flags', W_SRE_Pattern),
-    groupindex   = interp_attrproperty_w('w_groupindex', W_SRE_Pattern),
+    groupindex   = GetSetProperty(W_SRE_Pattern.fget_groupindex),
     groups       = interp_attrproperty('num_groups', W_SRE_Pattern),
     pattern      = interp_attrproperty_w('w_pattern', W_SRE_Pattern),
 )
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -53,6 +53,7 @@
         assert re.I | re.M == p.flags
         assert 2 == p.groups
         assert {"g": 2} == p.groupindex
+        raises(TypeError, "p.groupindex['g'] = 3")
 
     def test_repeat_minmax_overflow(self):
         import re
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -319,6 +319,11 @@
                 self, module=module, instance=instance,
                 strdict=strdict, kwargs=kwargs)
 
+    def newdictproxy(self, w_dict):
+        # e.g. for module/_sre/
+        from pypy.objspace.std.dictproxyobject import W_DictProxyObject
+        return W_DictProxyObject(w_dict)
+
     def newset(self, iterable_w=None):
         if iterable_w is None:
             return W_SetObject(self, None)

From pypy.commits at gmail.com  Tue Aug 30 10:30:07 2016
From: pypy.commits at gmail.com (Raemi)
Date: Tue, 30 Aug 2016 07:30:07 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: fix wrong graph generation by
 setting the return variable to NULL
Message-ID: <57c5986f.d42f1c0a.8eefb.765e@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: quad-color-gc
Changeset: r86744:626823d6d294
Date: 2016-08-30 16:27 +0200
http://bitbucket.org/pypy/pypy/changeset/626823d6d294/

Log:	fix wrong graph generation by setting the return variable to NULL

diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -1,5 +1,6 @@
 from rpython.rtyper.llannotation import SomePtr, SomeAddress, s_None
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper import rmodel
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.memory.gctransform.framework import (BaseFrameworkGCTransformer, BaseRootWalker)
 
@@ -69,8 +70,13 @@
 #                                  c_index, v_ptr])
 
     def gct_gc_fq_next_dead(self, hop):
-        pass
-#        index = self.get_finalizer_queue_index(hop)
+        # "return NULL" to tell PyPy that there are no finalizers to run (XXX)
+        op = hop.spaceop
+        null = lltype.nullptr(op.result.concretetype.TO)
+        c_null = rmodel.inputconst(op.result.concretetype, null)
+        hop.genop("same_as", [c_null], resultvar=op.result)
+
+        #        index = self.get_finalizer_queue_index(hop)
 #        c_ll_next_dead = self.finalizer_handlers[index][2]
 #        v_adr = hop.genop("direct_call", [c_ll_next_dead],
 #                          resulttype=llmemory.Address)

From pypy.commits at gmail.com  Tue Aug 30 10:30:09 2016
From: pypy.commits at gmail.com (Raemi)
Date: Tue, 30 Aug 2016 07:30:09 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: work around hard-coded
 pypy_header0 in funcgen.py
Message-ID: <57c59871.482cc20a.cce66.25d2@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: quad-color-gc
Changeset: r86745:8887dd513fe1
Date: 2016-08-30 16:29 +0200
http://bitbucket.org/pypy/pypy/changeset/8887dd513fe1/

Log:	work around hard-coded pypy_header0 in funcgen.py

	(should be fixed differently)

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -23,7 +23,7 @@
 
     TRANSLATION_PARAMS = {}
     HDR = lltype.Struct(
-            'pypyhdr_t',
+            'header',
             #('hdr', rffi.COpaque('object_t', hints={"is_qcgc_header": True})),
             ('flags', lltype.Signed),   # XXX: exploits knowledge about object_t
             ('tid', lltype.Signed),

From pypy.commits at gmail.com  Tue Aug 30 11:36:28 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 08:36:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: lots of details and corner
 cases to fix memoryview with the new attributes, 
Message-ID: <57c5a7fc.02c41c0a.7d8c2.9eec@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86746:a64fb8d7c7d8
Date: 2016-08-30 17:35 +0200
http://bitbucket.org/pypy/pypy/changeset/a64fb8d7c7d8/

Log:	lots of details and corner cases to fix memoryview with the new
	attributes, impl. buffer interface for array.array

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -636,6 +636,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -132,12 +132,16 @@
 
     def _copy_base(self, data, off):
         shapes = self.getshape()
-        step = shapes[0] // self.getitemsize()
+        step = shapes[0]
         strides = self.getstrides()
+        itemsize = self.getitemsize()
         for i in range(step):
-            bytes = self.buf.getslice(off, off+self.itemsize, 1, self.itemsize)
+            bytes = self.buf.getslice(off, off+itemsize, 1, itemsize)
             data.append(bytes)
             off += strides[0]
+            # do notcopy data if the sub buffer is out of bounds
+            if off >= self.buf.getlength():
+                break
 
     def getlength(self):
         if self.length != -1:
@@ -265,11 +269,9 @@
                 fmtiter = UnpackFormatIterator(space, buf)
                 fmtiter.interpret(self.format)
                 return fmtiter.result_w[0]
-        elif step == 1:
+        elif step >= 1:
             buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf, self.getformat(), itemsize)
-        else:
-            mv = W_MemoryView.copy(self)
+            mv = W_MemoryView.copy(self, buf)
             mv.slice(start, stop, step, size)
             mv.length = mv.bytecount_from_shape()
             mv._init_flags()
@@ -281,12 +283,11 @@
         # TODO subbuffer
         strides = self.getstrides()[:]
         shape = self.getshape()[:]
-        itemsize = self.itemsize
+        itemsize = self.getitemsize()
         dim = 0
-        length = self.buf.getlength()
-        self.buf = SubBuffer(self.buf, strides[dim] * start, size)
+        self.buf = SubBuffer(self.buf, strides[dim] * (start//itemsize), size * itemsize)
         shape[dim] = size
-        strides[dim] = strides[dim] * step * itemsize
+        strides[dim] = strides[dim] * step
         self.strides = strides
         self.shape = shape
 
@@ -299,12 +300,13 @@
         return length * self.getitemsize()
 
     @staticmethod
-    def copy(view):
+    def copy(view, buf=None):
         # TODO suboffsets
-        return W_MemoryView(view.buf, view.getformat(), view.getitemsize(),
+        if buf == None:
+            buf = view.buf
+        return W_MemoryView(buf, view.getformat(), view.getitemsize(),
                             view.getndim(), view.getshape()[:], view.getstrides()[:])
 
-
     def _apply_itemsize(self, space, start, size, itemsize):
         if itemsize > 1:
             start *= itemsize
@@ -351,17 +353,21 @@
                 raise oefmt(space.w_NotImplementedError,
                         "memoryview slice assignments are currently "
                         "restricted to ndim = 1")
+            # this is the case of a one dimensional copy!
+            # NOTE we could maybe make use of copy_base, but currently we do not
             itemsize = self.getitemsize()
             data = []
             src = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
             dst_strides = self.getstrides()
             dim = 0
-            dst = SubBuffer(self.buf, start + dst_strides[dim] * (start // itemsize), self.buf.getlength())
+            dst = SubBuffer(self.buf, start, size)
             src_stride0 = dst_strides[dim]
 
             off = 0
-            src_shape0 = size
+            src_shape0 = size // itemsize
             src_stride0 = src.getstrides()[0]
+            if isinstance(w_obj, W_MemoryView):
+                src_stride0 = w_obj.getstrides()[0]
             for i in range(src_shape0):
                 data.append(src.getslice(off,off+itemsize,1,itemsize))
                 off += src_stride0
@@ -575,8 +581,8 @@
         self.format = newfmt
         self.itemsize = itemsize
         self.ndim = 1
-        self.shape = [buf.getlength() // buf.getitemsize()]
-        self.strides = [buf.getitemsize()]
+        self.shape = [buf.getlength() // itemsize]
+        self.strides = [itemsize]
         # XX suboffsets
 
         self._init_flags()
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -225,7 +225,7 @@
         data = bytearray(b'abcdefghij')
         m3 = memoryview(data).cast('h')
         m3[1:5:2] = memoryview(b"xyXY").cast('h')
-        assert data == bytearray(b'abxyefXYij')
+        assert data == bytearray(eval("b'abxyefXYij'"))
 
 class MockBuffer(Buffer):
     def __init__(self, space, w_arr, w_dim, w_fmt, \

From pypy.commits at gmail.com  Tue Aug 30 11:56:52 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 08:56:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: apply small fixes. passing all
 memory view tests
Message-ID: <57c5acc4.94071c0a.1c406.9970@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86747:ff89f145aa44
Date: 2016-08-30 17:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ff89f145aa44/

Log:	apply small fixes. passing all memory view tests

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -269,9 +269,13 @@
                 fmtiter = UnpackFormatIterator(space, buf)
                 fmtiter.interpret(self.format)
                 return fmtiter.result_w[0]
-        elif step >= 1:
-            buf = SubBuffer(self.buf, start, size)
-            mv = W_MemoryView.copy(self, buf)
+        elif step == 1:
+            mv = W_MemoryView.copy(self)
+            mv.slice(start, stop, step, size)
+            mv._init_flags()
+            return mv
+        else:
+            mv = W_MemoryView.copy(self)
             mv.slice(start, stop, step, size)
             mv.length = mv.bytecount_from_shape()
             mv._init_flags()
@@ -279,13 +283,13 @@
 
     def slice(self, start, stop, step, size):
         # modifies the buffer, shape and stride to allow step to be > 1
-        # NOTE that start & stop are already byte offsets
+        # NOTE that start, stop & size are already byte offsets/count
         # TODO subbuffer
         strides = self.getstrides()[:]
         shape = self.getshape()[:]
         itemsize = self.getitemsize()
         dim = 0
-        self.buf = SubBuffer(self.buf, strides[dim] * (start//itemsize), size * itemsize)
+        self.buf = SubBuffer(self.buf, strides[dim] * (start//itemsize), size*step)
         shape[dim] = size
         strides[dim] = strides[dim] * step
         self.strides = strides

From pypy.commits at gmail.com  Tue Aug 30 12:31:55 2016
From: pypy.commits at gmail.com (vext01)
Date: Tue, 30 Aug 2016 09:31:55 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Another W^X site.
Message-ID: <57c5b4fb.8a13c20a.23ed6.605d@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86748:bb2143405c5f
Date: 2016-08-30 17:31 +0100
http://bitbucket.org/pypy/pypy/changeset/bb2143405c5f/

Log:	Another W^X site.

diff --git a/rpython/jit/backend/llsupport/gcreftracer.py b/rpython/jit/backend/llsupport/gcreftracer.py
--- a/rpython/jit/backend/llsupport/gcreftracer.py
+++ b/rpython/jit/backend/llsupport/gcreftracer.py
@@ -1,4 +1,4 @@
-from rpython.rlib import rgc
+from rpython.rlib import rgc, rmmap
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.jit.backend.llsupport.symbolic import WORD
@@ -33,10 +33,13 @@
     tr.array_base_addr = array_base_addr
     tr.array_length = length
     i = 0
+    array_base_addr_p = rffi.cast(rffi.CCHARP, array_base_addr)
+    rmmap.set_pages_writable(array_base_addr_p, length)
     while i < length:
         p = rffi.cast(rffi.SIGNEDP, array_base_addr + i * WORD)
         p[0] = rffi.cast(lltype.Signed, gcrefs[i])
         i += 1
+    rmmap.set_pages_executable(array_base_addr_p, length)
     llop.gc_writebarrier(lltype.Void, tr)
     # --no GC until here--
     return tr

From pypy.commits at gmail.com  Tue Aug 30 13:26:49 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 30 Aug 2016 10:26:49 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix translation when there is no
 SOCK_CLOEXEC
Message-ID: <57c5c1d9.c62f1c0a.57e03.b9cf@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86749:24d11640c483
Date: 2016-08-30 18:26 +0100
http://bitbucket.org/pypy/pypy/changeset/24d11640c483/

Log:	Fix translation when there is no SOCK_CLOEXEC

diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -526,7 +526,7 @@
                  fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socket() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.
@@ -655,7 +655,7 @@
         address, addr_p, addrlen_p = self._addrbuf()
         try:
             remove_inheritable = not inheritable
-            if (not inheritable and SOCK_CLOEXEC is not None
+            if (not inheritable and 'SOCK_CLOEXEC' in constants
                     and _c.HAVE_ACCEPT4
                     and _accept4_syscall.attempt_syscall()):
                 newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
@@ -1138,7 +1138,7 @@
         try:
             res = -1
             remove_inheritable = not inheritable
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socketpair() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.

From pypy.commits at gmail.com  Tue Aug 30 14:02:27 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 30 Aug 2016 11:02:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c5ca33.81a2c20a.cf2ff.8ee2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86750:2498fb9ffc58
Date: 2016-08-30 19:01 +0100
http://bitbucket.org/pypy/pypy/changeset/2498fb9ffc58/

Log:	hg merge default

diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -27,6 +27,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -114,7 +122,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -303,6 +303,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -526,7 +526,7 @@
                  fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socket() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.
@@ -655,7 +655,7 @@
         address, addr_p, addrlen_p = self._addrbuf()
         try:
             remove_inheritable = not inheritable
-            if (not inheritable and SOCK_CLOEXEC is not None
+            if (not inheritable and 'SOCK_CLOEXEC' in constants
                     and _c.HAVE_ACCEPT4
                     and _accept4_syscall.attempt_syscall()):
                 newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
@@ -1138,7 +1138,7 @@
         try:
             res = -1
             remove_inheritable = not inheritable
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socketpair() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.

From pypy.commits at gmail.com  Tue Aug 30 14:33:21 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 30 Aug 2016 11:33:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix test on -A
Message-ID: <57c5d171.c75dc20a.8a59a.9e2c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86751:b1dd05564a1c
Date: 2016-08-30 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/b1dd05564a1c/

Log:	Fix test on -A

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -850,6 +850,9 @@
         raises(TypeError, delattr, A(), 42)
 
     def test_getattr_None(self):
+        import sys
+        if '__pypy__' not in sys.modules:
+            skip('CPython uses wrapper types for this')
         from types import FunctionType, MethodType
         assert isinstance(getattr(type(None), '__eq__'), FunctionType)
         assert isinstance(getattr(None, '__eq__'), MethodType)
@@ -864,4 +867,3 @@
         assert isinstance(getattr(a, '__eq__'), MethodType)
         a.__eq__ = 42
         assert a.__eq__ == 42
-

From pypy.commits at gmail.com  Tue Aug 30 14:48:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 11:48:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge py3.5 memoryview additions
Message-ID: <57c5d51b.a719c20a.7a270.9d6e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86752:b63a89f9d9fd
Date: 2016-08-30 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/b63a89f9d9fd/

Log:	merge py3.5 memoryview additions

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -636,6 +636,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/module/micronumpy/test/__init__.py b/pypy/module/micronumpy/test/__init__.py
--- a/pypy/module/micronumpy/test/__init__.py
+++ b/pypy/module/micronumpy/test/__init__.py
@@ -1,2 +1,2 @@
 import py
-py.test.py3k_skip('not yet supported')
+# XXX py.test.py3k_skip('not yet supported')
diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -20,7 +20,9 @@
 for t in types:
     globals()[t] = dtype(t).type
 
-types = ['bool', 'int', 'float', 'complex', 'str', 'string', 'unicode', 'object']
+# removed 'string' and 'unicode' from that list to handle an error in
+# make_new_dtype (micronumpy/descriptor.py)
+types = ['bool', 'int', 'float', 'complex', 'str', 'object']
 for t in types:
     globals()[t + '_'] = dtype(t).type
 del types
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -9,7 +9,7 @@
 
     @classmethod
     def setup_class(cls):
-        py.test.py3k_skip("micronumpy not supported on py3k")
+        # XXX py.test.py3k_skip("micronumpy not supported on py3k")
         if option.runappdirect:
             import sys
             if '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -502,6 +502,7 @@
     return data
 
 HEXDIGITS = "0123456789abcdef"
+PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8-1)-1)
 
 @specialize.arg(3) # raw access
 def _array_to_hexstring(space, buf, len=0, rawaccess=False):
@@ -509,11 +510,11 @@
         length = len
     else:
         length = buf.getlength()
+    hexstring = StringBuilder(length*2)
 
-    if length > sys.maxint / 2:
+    if length > PY_SIZE_T_MAX/2:
         raise OperationError(space.w_MemoryError, space.w_None)
 
-    hexstring = StringBuilder(length*2)
     for i in range(length):
         if rawaccess:
             byte = ord(buf[i])
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -659,6 +659,10 @@
     def descr_upper(self, space):
         return W_BytesObject(self._value.upper())
 
+    def descr_hex(self, space):
+        from pypy.objspace.std.bytearrayobject import _array_to_hexstring
+        return _array_to_hexstring(space, StringBuffer(self._value))
+
     @staticmethod
     def _iter_getitem_result(self, space, index):
         assert isinstance(self, W_BytesObject)
@@ -843,6 +847,7 @@
 
     fromhex = interp2app(W_BytesObject.descr_fromhex, as_classmethod=True),
     maketrans = interp2app(W_BytesObject.descr_maketrans, as_classmethod=True),
+    hex = interp2app(W_BytesObject.descr_hex),
 )
 W_BytesObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -10,27 +10,73 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
+from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
 from pypy.objspace.std.bytesobject import getbytevalue
-from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
+from rpython.rlib.unroll import unrolling_iterable
+
+MEMORYVIEW_MAX_DIM = 64
+MEMORYVIEW_SCALAR   = 0x0001
+MEMORYVIEW_C        = 0x0002
+MEMORYVIEW_FORTRAN  = 0x0004
+MEMORYVIEW_SCALAR   = 0x0008
+MEMORYVIEW_PIL      = 0x0010
 
 
 class W_MemoryView(W_Root):
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
-    _immutable_fields_ = ['format', 'itemsize']
 
-    def __init__(self, buf, format='B', itemsize=1):
+    def __init__(self, buf, format=None, itemsize=1, ndim=-1,
+                 shape=None, strides=None, suboffsets=None):
         assert isinstance(buf, Buffer)
         self.buf = buf
         self._hash = -1
+        # private copies of format, shape, itemsize, ... on this class
         self.format = format
         self.itemsize = itemsize
+        self.shape = shape
+        self.strides = strides
+        self.suboffsets = suboffsets
+        self.ndim = ndim
+        self.flags = 0
+        self.length = -1
+        self._init_flags()
+
+    # several fields are "overwritten" by the memory view (shape, strides, ...)
+    # thus use only those getter fields instead of directly accessing the fields
+    def getndim(self):
+        if self.ndim == -1:
+            return self.buf.getndim()
+        return self.ndim
+
+    def getshape(self):
+        if self.shape is None:
+            return self.buf.getshape()
+        return self.shape
+
+    def getstrides(self):
+        if self.strides is None:
+            return self.buf.getstrides()
+        return self.strides
+
+    def getitemsize(self):
+        return self.itemsize
+
+    # memoryview needs to modify the field 'format', to prevent the modification
+    # of the buffer, we save the new format here!
+    def getformat(self):
+        if self.format is None:
+            return self.buf.getformat()
+        return self.format
+
+    def setformat(self, value):
+        self.format = value
 
     def buffer_w_ex(self, space, flags):
         self._check_released(space)
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf, self.format, self.itemsize
+        return self.buf, self.getformat(), self.itemsize
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
@@ -63,10 +109,43 @@
     descr_ne = _make_descr__cmp('ne')
 
     def as_str(self):
+        return ''.join(self.copy_buffer())
+
+    def copy_buffer(self):
         buf = self.buf
-        return buf.as_str()
+        n_bytes = buf.getlength()
+        data = []
+        self._copy_rec(0, data, 0)
+        return data
+
+    def _copy_rec(self, idim, data, off):
+        shapes = self.getshape()
+        shape = shapes[idim]
+
+        if self.getndim()-1 == idim:
+            self._copy_base(data,off)
+            return
+
+        for i in range(shape):
+            self._copy_rec(idim+1,data,off)
+            off += strides[idim]
+
+    def _copy_base(self, data, off):
+        shapes = self.getshape()
+        step = shapes[0]
+        strides = self.getstrides()
+        itemsize = self.getitemsize()
+        for i in range(step):
+            bytes = self.buf.getslice(off, off+itemsize, 1, itemsize)
+            data.append(bytes)
+            off += strides[0]
+            # do notcopy data if the sub buffer is out of bounds
+            if off >= self.buf.getlength():
+                break
 
     def getlength(self):
+        if self.length != -1:
+            return self.length // self.itemsize
         return self.buf.getlength() // self.itemsize
 
     def descr_tobytes(self, space):
@@ -75,43 +154,184 @@
 
     def descr_tolist(self, space):
         self._check_released(space)
+
+        buf = self.buf
+        dim = self.getndim()
+        fmt = self.getformat()
+        if dim == 0:
+            raise NotImplementedError
+        elif dim == 1:
+            itemsize = self.getitemsize()
+            return self._tolist(space, buf, buf.getlength() // itemsize, fmt)
+        else:
+            return self._tolist_rec(space, buf, 0, 0, fmt)
+
+    def _tolist(self, space, buf, count, fmt):
         # TODO: this probably isn't very fast
-        fmtiter = UnpackFormatIterator(space, self.buf)
-        fmtiter.interpret(self.format * self.getlength())
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(fmt * count)
         return space.newlist(fmtiter.result_w)
 
+    def _tolist_rec(self, space, buf, start, idim, fmt):
+        strides = self.getstrides()
+        shape = self.getshape()
+        #
+        dim = idim+1
+        stride = strides[idim]
+        itemsize = self.getitemsize()
+        dimshape = shape[idim]
+        #
+        if dim >= self.getndim():
+            bytecount = (stride * dimshape)
+            count = bytecount // itemsize
+            return self._tolist(space, buf, count, fmt)
+        items = [None] * dimshape
+
+        for i in range(dimshape):
+            item = self._tolist_rec(space, SubBuffer(buf, start, stride), start, idim+1, fmt)
+            items[i] = item
+            start += stride
+
+        return space.newlist(items)
+
+
+    def _start_from_tuple(self, space, w_tuple):
+        from pypy.objspace.std.tupleobject import W_TupleObject
+        start = 0
+
+        view = self.buf
+        length = space.len_w(w_tuple)
+        dim = view.getndim()
+        dim = 0
+        assert isinstance(w_tuple, W_TupleObject)
+        while dim < length:
+            w_obj = w_tuple.getitem(space, dim)
+            index = w_obj.int_w(space)
+            start = self.lookup_dimension(space, start, dim, index)
+            dim += 1
+        return start
+
+    def lookup_dimension(self, space, start, dim, index):
+        view = self.buf
+        shape = view.getshape()
+        strides = view.getstrides()
+        nitems = shape[dim]
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim+1)
+        start += strides[dim] * index
+        # TODO suboffsets?
+        return start
+
+    def _getitem_tuple_indexed(self, space, w_index):
+        view = self.buf
+
+        fmt = view.getformat() # TODO adjust format?
+
+        length = space.len_w(w_index)
+        ndim = view.getndim()
+        if length < ndim:
+            raise OperationError(space.w_NotImplementedError, \
+                    space.wrap("sub-views are not implemented"))
+
+        if length > ndim:
+            raise oefmt(space.w_TypeError, \
+                    "cannot index %d-dimension view with %d-element tuple",
+                    length, ndim)
+
+        start = self._start_from_tuple(space, w_index)
+
+        buf = SubBuffer(self.buf, start, view.getitemsize())
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(fmt)
+        return fmtiter.result_w[0]
+
+
     def descr_getitem(self, space, w_index):
         self._check_released(space)
+
+        if space.isinstance_w(w_index, space.w_tuple):
+            return self._getitem_tuple_indexed(space, w_index)
+
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
-        itemsize = self.itemsize
+        itemsize = self.getitemsize()
+        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
             if itemsize == 1:
                 ch = self.buf.getitem(start)
                 return space.newint(ord(ch))
             else:
                 # TODO: this probably isn't very fast
-                buf = SubBuffer(self.buf, start * itemsize, itemsize)
+                buf = SubBuffer(self.buf, start, itemsize)
                 fmtiter = UnpackFormatIterator(space, buf)
                 fmtiter.interpret(self.format)
                 return fmtiter.result_w[0]
         elif step == 1:
-            buf = SubBuffer(self.buf, start * itemsize, size * itemsize)
-            return W_MemoryView(buf, self.format, itemsize)
+            mv = W_MemoryView.copy(self)
+            mv.slice(start, stop, step, size)
+            mv._init_flags()
+            return mv
         else:
-            # XXX needs to return a W_MemoryView with a NonContiguousSubBuffer
-            # maybe?  Need to check the cpyext requirements for that
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX extended slicing")
+            mv = W_MemoryView.copy(self)
+            mv.slice(start, stop, step, size)
+            mv.length = mv.bytecount_from_shape()
+            mv._init_flags()
+            return mv
+
+    def slice(self, start, stop, step, size):
+        # modifies the buffer, shape and stride to allow step to be > 1
+        # NOTE that start, stop & size are already byte offsets/count
+        # TODO subbuffer
+        strides = self.getstrides()[:]
+        shape = self.getshape()[:]
+        itemsize = self.getitemsize()
+        dim = 0
+        self.buf = SubBuffer(self.buf, strides[dim] * (start//itemsize), size*step)
+        shape[dim] = size
+        strides[dim] = strides[dim] * step
+        self.strides = strides
+        self.shape = shape
+
+    def bytecount_from_shape(self):
+        dim = self.getndim()
+        shape = self.getshape()
+        length = 1
+        for i in range(dim):
+            length *= shape[i]
+        return length * self.getitemsize()
+
+    @staticmethod
+    def copy(view, buf=None):
+        # TODO suboffsets
+        if buf == None:
+            buf = view.buf
+        return W_MemoryView(buf, view.getformat(), view.getitemsize(),
+                            view.getndim(), view.getshape()[:], view.getstrides()[:])
+
+    def _apply_itemsize(self, space, start, size, itemsize):
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+
+        stop  = start + size
+        # start & stop are now byte offset, thus use self.buf.getlength()
+        if stop > self.buf.getlength():
+            raise oefmt(space.w_IndexError, 'index out of range')
+
+        return start, stop, size
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         if space.isinstance_w(w_index, space.w_tuple):
-            raise oefmt(space.w_NotImplementedError, "XXX tuple setitem")
+            raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.itemsize
+        itemsize = self.getitemsize()
+        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
             if itemsize == 1:
                 ch = getbytevalue(space, w_obj)
@@ -125,16 +345,41 @@
                     raise oefmt(space.w_TypeError,
                                 "memoryview: invalid type for format '%s'",
                                 self.format)
-                self.buf.setslice(start * itemsize, fmtiter.result.build())
+                self.buf.setslice(start, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-            if value.getlength() != size * self.itemsize:
+            if value.getlength() != size:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
-            self.buf.setslice(start * itemsize, value.as_str())
+            self.buf.setslice(start, value.as_str())
         else:
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX extended slicing")
+            if self.getndim() != 1:
+                raise oefmt(space.w_NotImplementedError,
+                        "memoryview slice assignments are currently "
+                        "restricted to ndim = 1")
+            # this is the case of a one dimensional copy!
+            # NOTE we could maybe make use of copy_base, but currently we do not
+            itemsize = self.getitemsize()
+            data = []
+            src = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+            dst_strides = self.getstrides()
+            dim = 0
+            dst = SubBuffer(self.buf, start, size)
+            src_stride0 = dst_strides[dim]
+
+            off = 0
+            src_shape0 = size // itemsize
+            src_stride0 = src.getstrides()[0]
+            if isinstance(w_obj, W_MemoryView):
+                src_stride0 = w_obj.getstrides()[0]
+            for i in range(src_shape0):
+                data.append(src.getslice(off,off+itemsize,1,itemsize))
+                off += src_stride0
+            off = 0
+            dst_stride0 = self.getstrides()[0] * step
+            for dataslice in data:
+                dst.setslice(off, dataslice)
+                off += dst_stride0
 
     def descr_len(self, space):
         self._check_released(space)
@@ -142,11 +387,11 @@
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap(self.format)
+        return space.wrap(self.getformat())
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.newint(self.itemsize)
+        return space.wrap(self.itemsize)
 
     def w_get_ndim(self, space):
         self._check_released(space)
@@ -158,11 +403,11 @@
 
     def w_get_shape(self, space):
         self._check_released(space)
-        return space.newtuple([space.newint(self.getlength())])
+        return space.newtuple([space.wrap(x) for x in self.getshape()])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.newint(self.itemsize)])
+        return space.newtuple([space.wrap(x) for x in self.getstrides()])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)
@@ -240,20 +485,179 @@
             size = rffi.sizeof(rffi.VOIDP)
         return size
 
+    def _zero_in_shape(self):
+        # this method could be moved to the class Buffer
+        buf = self.buf
+        shape = buf.getshape()
+        for i in range(buf.getndim()):
+            if shape[i] == 0:
+                return True
+        return False
+
     def descr_cast(self, space, w_format, w_shape=None):
         self._check_released(space)
-        if not space.is_none(w_shape):
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX cast() with a shape")
+
+        if not space.isinstance_w(w_format, space.w_unicode):
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: format argument must be a string"))
+
         fmt = space.str_w(w_format)
-        newitemsize = self.get_native_fmtchar(fmt)
-        return W_MemoryView(self.buf, fmt, newitemsize)
+        buf = self.buf
+        ndim = 1
+
+        if not memory_view_c_contiguous(space, self.flags):
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: casts are restricted" \
+                               " to C-contiguous views"))
+
+        if (w_shape or buf.getndim() != 1) and self._zero_in_shape():
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: cannot casts view with" \
+                               " zeros in shape or strides"))
+
+        itemsize = self.get_native_fmtchar(fmt)
+        if w_shape:
+            if not (space.isinstance_w(w_shape, space.w_list) or space.isinstance_w(w_shape, space.w_tuple)):
+                raise oefmt(space.w_TypeError, "expected list or tuple got %T", w_shape)
+            ndim = space.len_w(w_shape)
+            if ndim > MEMORYVIEW_MAX_DIM:
+                raise oefmt(space.w_ValueError, \
+                        "memoryview: number of dimensions must not exceed %d",
+                        ndim)
+            # yes access ndim as field
+            if self.ndim > 1 and buf.getndim() != 1:
+                raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: cast must be 1D -> ND or ND -> 1D"))
+
+        mv = W_MemoryView(buf, self.format, self.itemsize)
+        origfmt = mv.getformat()
+        mv._cast_to_1D(space, origfmt, fmt, itemsize)
+        if w_shape:
+            fview = space.fixedview(w_shape)
+            shape = [space.int_w(w_obj) for w_obj in fview]
+            mv._cast_to_ND(space, shape, ndim)
+        return mv
+
+    def _init_flags(self):
+        buf = self.buf
+        ndim = buf.getndim()
+        flags = 0
+        if ndim == 0:
+            flags |= MEMORYVIEW_SCALAR | MEMORYVIEW_C | MEMORYVIEW_FORTRAN
+        if ndim == 1:
+            shape = buf.getshape()
+            strides = buf.getstrides()
+            if len(shape) > 0 and shape[0] == 1 and \
+               len(strides) > 0 and strides[0] == buf.getitemsize():
+                flags |= MEMORYVIEW_C | MEMORYVIEW_SCALAR
+        # TODO for now?
+        flags |= MEMORYVIEW_C
+        # TODO if buf.is_contiguous('C'):
+        # TODO     flags |= MEMORYVIEW_C
+        # TODO elif buf.is_contiguous('F'):
+        # TODO     flags |= MEMORYVIEW_FORTRAN
+
+        # TODO missing suboffsets
+
+        self.flags = flags
+
+    def _cast_to_1D(self, space, origfmt, fmt, itemsize):
+        buf = self.buf
+        if itemsize < 0:
+            raise oefmt(space.w_ValueError, "memoryview: destination" \
+                    " format must be a native single character format prefixed" \
+                    " with an optional '@'")
+
+        if self.get_native_fmtchar(origfmt) < 0 or \
+           (not is_byte_format(fmt) and not is_byte_format(origfmt)):
+            raise oefmt(space.w_TypeError,
+                    "memoryview: cannot cast between" \
+                    " two non-byte formats")
+
+        if buf.getlength() % itemsize != 0:
+            raise oefmt(space.w_TypeError,
+                    "memoryview: length is not a multiple of itemsize")
+
+        newfmt = self.get_native_fmtstr(fmt)
+        if not newfmt:
+            raise oefmt(space.w_RuntimeError,
+                    "memoryview: internal error")
+        self.format = newfmt
+        self.itemsize = itemsize
+        self.ndim = 1
+        self.shape = [buf.getlength() // itemsize]
+        self.strides = [itemsize]
+        # XX suboffsets
+
+        self._init_flags()
+
+    def get_native_fmtstr(self, fmt):
+        lenfmt = len(fmt)
+        nat = False
+        if lenfmt == 0:
+            return None
+        elif lenfmt == 1:
+            format = fmt[0] # fine!
+        elif lenfmt == 2:
+            if fmt[0] == '@':
+                nat = True
+                format = fmt[1]
+            else:
+                return None
+        else:
+            return None
+
+        chars = ['c','b','B','h','H','i','I','l','L','q',
+                 'Q','n','N','f','d','?','P']
+        for c in unrolling_iterable(chars):
+            if c == format:
+                if nat: return '@'+c
+                else: return c
+
+        return None
+
+    def _cast_to_ND(self, space, shape, ndim):
+        buf = self.buf
+
+        self.ndim = ndim
+        length = self.itemsize
+        if ndim == 0:
+            self.shape = []
+            self.strides = []
+        else:
+            self.shape = shape
+            for i in range(ndim):
+                length *= shape[i]
+            self._init_strides_from_shape()
+
+        if length != self.buf.getlength():
+            raise OperationError(space.w_TypeError,
+                    space.wrap("memoryview: product(shape) * itemsize != buffer size"))
+
+        self._init_flags()
+
+    def _init_strides_from_shape(self):
+        shape = self.getshape()
+        s = [0] * len(shape)
+        self.strides = s
+        ndim = self.getndim()
+        s[ndim-1] = self.itemsize
+        i = ndim-2
+        while i >= 0:
+            s[i] = s[i+1] * shape[i+1]
+            i -= 1
 
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         self._check_released(space)
         return _array_to_hexstring(space, self.buf)
 
+def is_byte_format(char):
+    return char == 'b' or char == 'B' or char == 'c'
+
+def memory_view_c_contiguous(space, flags):
+    return flags & (space.BUF_CONTIG_RO|MEMORYVIEW_C) != 0
+
 W_MemoryView.typedef = TypeDef(
     "memoryview",
     __doc__ = """\
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -1,3 +1,11 @@
+import py
+import struct
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.typedef import TypeDef
+from rpython.rlib.buffer import Buffer
+from pypy.conftest import option
+
 class AppTestMemoryView:
     spaceconfig = dict(usemodules=['array'])
 
@@ -38,7 +46,7 @@
         assert len(w) == 1
         assert list(w) == [97]
         v[::2] = b'ABC'
-        assert data == bytearray(b'AbBeCg')
+        assert data == bytearray(eval("b'AbBeCg'"))
 
     def test_memoryview_attrs(self):
         v = memoryview(b"a"*100)
@@ -217,4 +225,159 @@
         data = bytearray(b'abcdefghij')
         m3 = memoryview(data).cast('h')
         m3[1:5:2] = memoryview(b"xyXY").cast('h')
-        assert data == bytearray(b'abxyefXYij')
+        assert data == bytearray(eval("b'abxyefXYij'"))
+
+class MockBuffer(Buffer):
+    def __init__(self, space, w_arr, w_dim, w_fmt, \
+                 w_itemsize, w_strides, w_shape):
+        self.space = space
+        self.w_arr = w_arr
+        self.arr = []
+        self.ndim = space.int_w(w_dim)
+        self.format = space.str_w(w_fmt)
+        self.itemsize = space.int_w(w_itemsize)
+        self.strides = []
+        for w_i in w_strides.getitems_unroll():
+            self.strides.append(space.int_w(w_i))
+        self.shape = []
+        for w_i in w_shape.getitems_unroll():
+            self.shape.append(space.int_w(w_i))
+        self.readonly = True
+        self.shape.append(space.len_w(w_arr))
+        self.data = []
+        itemsize = 1
+        worklist = [(1,w_arr)]
+        while worklist:
+            dim, w_work = worklist.pop()
+            if space.isinstance_w(w_work, space.w_list):
+                for j, w_obj in enumerate(w_work.getitems_unroll()):
+                    worklist.insert(0, (dim+1, w_obj))
+                continue
+            byte = struct.pack(self.format, space.int_w(w_work))
+            for c in byte:
+                self.data.append(c)
+        self.data = ''.join(self.data)
+
+    def getformat(self):
+        return self.format
+
+    def getitem(self, index):
+        return self.data[index:index+1]
+
+    def getlength(self):
+        return len(self.data)
+
+    def getitemsize(self):
+        return self.itemsize
+
+    def getndim(self):
+        return self.ndim
+
+    def getstrides(self):
+        return self.strides
+
+    def getshape(self):
+        return self.shape
+
+    def is_contiguous(self, format):
+        return format == 'C'
+
+class W_MockArray(W_Root):
+    def __init__(self, w_list, w_dim, w_fmt, w_size, w_strides, w_shape):
+        self.w_list = w_list
+        self.w_dim = w_dim
+        self.w_fmt = w_fmt
+        self.w_size = w_size
+        self.w_strides = w_strides
+        self.w_shape = w_shape
+
+    @staticmethod
+    def descr_new(space, w_type, w_list, w_dim, w_fmt, \
+                         w_size, w_strides, w_shape):
+        return W_MockArray(w_list, w_dim, w_fmt, w_size, w_strides, w_shape)
+
+    def buffer_w(self, space, flags):
+        return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
+                          self.w_size, self.w_strides, self.w_shape)
+
+    def buffer_w_ex(self, space, flags):
+        return self.buffer_w(space, flags), space.str_w(self.w_fmt), space.int_w(self.w_size)
+
+W_MockArray.typedef = TypeDef("MockArray",
+    __new__ = interp2app(W_MockArray.descr_new),
+)
+
+class AppTestMemoryViewMockBuffer(object):
+    spaceconfig = dict(usemodules=[])
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("Impossible to run on appdirect")
+        cls.w_MockArray = cls.space.gettypefor(W_MockArray)
+
+    def test_tuple_indexing(self):
+        content = self.MockArray([[0,1,2,3], [4,5,6,7], [8,9,10,11]],
+                                 dim=2, fmt='B', size=1,
+                                 strides=[4,1], shape=[3,4])
+        view = memoryview(content)
+        assert view[0,0] == 0
+        assert view[2,0] == 8
+        assert view[2,3] == 11
+        assert view[-1,-1] == 11
+        assert view[-3,-4] == 0
+
+        raises(IndexError, "view.__getitem__((2**63-1,0))")
+        raises(TypeError, "view.__getitem__((0, 0, 0))")
+
+    def test_tuple_indexing_int(self):
+        content = self.MockArray([ [[1],[2],[3]], [[4],[5],[6]] ],
+                                 dim=3, fmt='i', size=4,
+                                 strides=[12,4,4], shape=[2,3,1])
+        view = memoryview(content)
+        assert view[0,0,0] == 1
+        assert view[-1,2,0] == 6
+
+    def test_cast_non_byte(self):
+        empty = self.MockArray([], dim=1, fmt='i', size=4, strides=[1], shape=[1])
+        view = memoryview(empty)
+        raises(TypeError, "view.cast('l')")
+        try:
+            view.cast('l')
+            assert False, "i -> l not possible. buffer must be byte format"
+        except TypeError:
+            pass
+
+    def test_cast_empty(self):
+        empty = self.MockArray([], dim=1, fmt='b', size=1, strides=[1], shape=[1])
+        view = memoryview(empty)
+        cview = view.cast('i')
+        assert cview.tobytes() == b''
+        assert cview.tolist() == []
+        assert view.format == 'b'
+        assert cview.format == 'i'
+        #
+        assert cview.cast('b').cast('q').cast('b').tolist() == []
+        #
+        assert cview.format == 'i'
+        raises(TypeError, "cview.cast('i')")
+
+    def test_cast_with_shape(self):
+        empty = self.MockArray([1,0,2,0,3,0],
+                    dim=1, fmt='h', size=2,
+                    strides=[8], shape=[6])
+        view = memoryview(empty)
+        byteview = view.cast('b')
+        assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]
+        i32view = byteview.cast('i', shape=[1,3])
+        assert i32view.format == 'i'
+        assert i32view.itemsize == 4
+        assert i32view.tolist() == [[1,2,3]]
+        i32view = byteview.cast('i', shape=(1,3))
+        assert i32view.tolist() == [[1,2,3]]
+
+    def test_cast_bytes(self):
+        bytes = b"\x02\x00\x03\x00\x04\x00" \
+                b"\x05\x00\x06\x00\x07\x00"
+        view = memoryview(bytes)
+        v = view.cast('h', shape=(3,2))
+        assert v.tolist() == [[2,3],[4,5],[6,7]]
+        raises(TypeError, "view.cast('h', shape=(3,3))")

From pypy.commits at gmail.com  Tue Aug 30 14:49:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 11:49:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge translation fix
Message-ID: <57c5d51e.c3f0c20a.cb08.9ebc@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86754:21d53f6805c2
Date: 2016-08-30 20:46 +0200
http://bitbucket.org/pypy/pypy/changeset/21d53f6805c2/

Log:	merge translation fix

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -636,6 +636,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/module/micronumpy/test/__init__.py b/pypy/module/micronumpy/test/__init__.py
--- a/pypy/module/micronumpy/test/__init__.py
+++ b/pypy/module/micronumpy/test/__init__.py
@@ -1,2 +1,2 @@
 import py
-py.test.py3k_skip('not yet supported')
+# XXX py.test.py3k_skip('not yet supported')
diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -20,7 +20,9 @@
 for t in types:
     globals()[t] = dtype(t).type
 
-types = ['bool', 'int', 'float', 'complex', 'str', 'string', 'unicode', 'object']
+# removed 'string' and 'unicode' from that list to handle an error in
+# make_new_dtype (micronumpy/descriptor.py)
+types = ['bool', 'int', 'float', 'complex', 'str', 'object']
 for t in types:
     globals()[t + '_'] = dtype(t).type
 del types
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -9,7 +9,7 @@
 
     @classmethod
     def setup_class(cls):
-        py.test.py3k_skip("micronumpy not supported on py3k")
+        # XXX py.test.py3k_skip("micronumpy not supported on py3k")
         if option.runappdirect:
             import sys
             if '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -502,6 +502,7 @@
     return data
 
 HEXDIGITS = "0123456789abcdef"
+PY_SIZE_T_MAX = intmask(2**(rffi.sizeof(rffi.SIZE_T)*8-1)-1)
 
 @specialize.arg(3) # raw access
 def _array_to_hexstring(space, buf, len=0, rawaccess=False):
@@ -509,11 +510,11 @@
         length = len
     else:
         length = buf.getlength()
+    hexstring = StringBuilder(length*2)
 
-    if length > sys.maxint / 2:
+    if length > PY_SIZE_T_MAX/2:
         raise OperationError(space.w_MemoryError, space.w_None)
 
-    hexstring = StringBuilder(length*2)
     for i in range(length):
         if rawaccess:
             byte = ord(buf[i])
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -659,6 +659,10 @@
     def descr_upper(self, space):
         return W_BytesObject(self._value.upper())
 
+    def descr_hex(self, space):
+        from pypy.objspace.std.bytearrayobject import _array_to_hexstring
+        return _array_to_hexstring(space, StringBuffer(self._value))
+
     @staticmethod
     def _iter_getitem_result(self, space, index):
         assert isinstance(self, W_BytesObject)
@@ -843,6 +847,7 @@
 
     fromhex = interp2app(W_BytesObject.descr_fromhex, as_classmethod=True),
     maketrans = interp2app(W_BytesObject.descr_maketrans, as_classmethod=True),
+    hex = interp2app(W_BytesObject.descr_hex),
 )
 W_BytesObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -10,27 +10,73 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
+from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
 from pypy.objspace.std.bytesobject import getbytevalue
-from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
+from rpython.rlib.unroll import unrolling_iterable
+
+MEMORYVIEW_MAX_DIM = 64
+MEMORYVIEW_SCALAR   = 0x0001
+MEMORYVIEW_C        = 0x0002
+MEMORYVIEW_FORTRAN  = 0x0004
+MEMORYVIEW_SCALAR   = 0x0008
+MEMORYVIEW_PIL      = 0x0010
 
 
 class W_MemoryView(W_Root):
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
-    _immutable_fields_ = ['format', 'itemsize']
 
-    def __init__(self, buf, format='B', itemsize=1):
+    def __init__(self, buf, format=None, itemsize=1, ndim=-1,
+                 shape=None, strides=None, suboffsets=None):
         assert isinstance(buf, Buffer)
         self.buf = buf
         self._hash = -1
+        # private copies of format, shape, itemsize, ... on this class
         self.format = format
         self.itemsize = itemsize
+        self.shape = shape
+        self.strides = strides
+        self.suboffsets = suboffsets
+        self.ndim = ndim
+        self.flags = 0
+        self.length = -1
+        self._init_flags()
+
+    # several fields are "overwritten" by the memory view (shape, strides, ...)
+    # thus use only those getter fields instead of directly accessing the fields
+    def getndim(self):
+        if self.ndim == -1:
+            return self.buf.getndim()
+        return self.ndim
+
+    def getshape(self):
+        if self.shape is None:
+            return self.buf.getshape()
+        return self.shape
+
+    def getstrides(self):
+        if self.strides is None:
+            return self.buf.getstrides()
+        return self.strides
+
+    def getitemsize(self):
+        return self.itemsize
+
+    # memoryview needs to modify the field 'format', to prevent the modification
+    # of the buffer, we save the new format here!
+    def getformat(self):
+        if self.format is None:
+            return self.buf.getformat()
+        return self.format
+
+    def setformat(self, value):
+        self.format = value
 
     def buffer_w_ex(self, space, flags):
         self._check_released(space)
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf, self.format, self.itemsize
+        return self.buf, self.getformat(), self.itemsize
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
@@ -63,10 +109,45 @@
     descr_ne = _make_descr__cmp('ne')
 
     def as_str(self):
+        return ''.join(self.copy_buffer())
+
+    def copy_buffer(self):
         buf = self.buf
-        return buf.as_str()
+        n_bytes = buf.getlength()
+        data = []
+        self._copy_rec(0, data, 0)
+        return data
+
+    def _copy_rec(self, idim, data, off):
+        shapes = self.getshape()
+        shape = shapes[idim]
+        strides = self.getstrides()
+
+        if self.getndim()-1 == idim:
+            self._copy_base(data,off)
+            return
+
+        # TODO add a test that has at least 2 dims
+        for i in range(shape):
+            self._copy_rec(idim+1,data,off)
+            off += strides[idim]
+
+    def _copy_base(self, data, off):
+        shapes = self.getshape()
+        step = shapes[0]
+        strides = self.getstrides()
+        itemsize = self.getitemsize()
+        for i in range(step):
+            bytes = self.buf.getslice(off, off+itemsize, 1, itemsize)
+            data.append(bytes)
+            off += strides[0]
+            # do notcopy data if the sub buffer is out of bounds
+            if off >= self.buf.getlength():
+                break
 
     def getlength(self):
+        if self.length != -1:
+            return self.length // self.itemsize
         return self.buf.getlength() // self.itemsize
 
     def descr_tobytes(self, space):
@@ -75,43 +156,184 @@
 
     def descr_tolist(self, space):
         self._check_released(space)
+
+        buf = self.buf
+        dim = self.getndim()
+        fmt = self.getformat()
+        if dim == 0:
+            raise NotImplementedError
+        elif dim == 1:
+            itemsize = self.getitemsize()
+            return self._tolist(space, buf, buf.getlength() // itemsize, fmt)
+        else:
+            return self._tolist_rec(space, buf, 0, 0, fmt)
+
+    def _tolist(self, space, buf, count, fmt):
         # TODO: this probably isn't very fast
-        fmtiter = UnpackFormatIterator(space, self.buf)
-        fmtiter.interpret(self.format * self.getlength())
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(fmt * count)
         return space.newlist(fmtiter.result_w)
 
+    def _tolist_rec(self, space, buf, start, idim, fmt):
+        strides = self.getstrides()
+        shape = self.getshape()
+        #
+        dim = idim+1
+        stride = strides[idim]
+        itemsize = self.getitemsize()
+        dimshape = shape[idim]
+        #
+        if dim >= self.getndim():
+            bytecount = (stride * dimshape)
+            count = bytecount // itemsize
+            return self._tolist(space, buf, count, fmt)
+        items = [None] * dimshape
+
+        for i in range(dimshape):
+            item = self._tolist_rec(space, SubBuffer(buf, start, stride), start, idim+1, fmt)
+            items[i] = item
+            start += stride
+
+        return space.newlist(items)
+
+
+    def _start_from_tuple(self, space, w_tuple):
+        from pypy.objspace.std.tupleobject import W_TupleObject
+        start = 0
+
+        view = self.buf
+        length = space.len_w(w_tuple)
+        dim = view.getndim()
+        dim = 0
+        assert isinstance(w_tuple, W_TupleObject)
+        while dim < length:
+            w_obj = w_tuple.getitem(space, dim)
+            index = w_obj.int_w(space)
+            start = self.lookup_dimension(space, start, dim, index)
+            dim += 1
+        return start
+
+    def lookup_dimension(self, space, start, dim, index):
+        view = self.buf
+        shape = view.getshape()
+        strides = view.getstrides()
+        nitems = shape[dim]
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim+1)
+        start += strides[dim] * index
+        # TODO suboffsets?
+        return start
+
+    def _getitem_tuple_indexed(self, space, w_index):
+        view = self.buf
+
+        fmt = view.getformat() # TODO adjust format?
+
+        length = space.len_w(w_index)
+        ndim = view.getndim()
+        if length < ndim:
+            raise OperationError(space.w_NotImplementedError, \
+                    space.wrap("sub-views are not implemented"))
+
+        if length > ndim:
+            raise oefmt(space.w_TypeError, \
+                    "cannot index %d-dimension view with %d-element tuple",
+                    length, ndim)
+
+        start = self._start_from_tuple(space, w_index)
+
+        buf = SubBuffer(self.buf, start, view.getitemsize())
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(fmt)
+        return fmtiter.result_w[0]
+
+
     def descr_getitem(self, space, w_index):
         self._check_released(space)
+
+        if space.isinstance_w(w_index, space.w_tuple):
+            return self._getitem_tuple_indexed(space, w_index)
+
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
-        itemsize = self.itemsize
+        itemsize = self.getitemsize()
+        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
             if itemsize == 1:
                 ch = self.buf.getitem(start)
                 return space.newint(ord(ch))
             else:
                 # TODO: this probably isn't very fast
-                buf = SubBuffer(self.buf, start * itemsize, itemsize)
+                buf = SubBuffer(self.buf, start, itemsize)
                 fmtiter = UnpackFormatIterator(space, buf)
                 fmtiter.interpret(self.format)
                 return fmtiter.result_w[0]
         elif step == 1:
-            buf = SubBuffer(self.buf, start * itemsize, size * itemsize)
-            return W_MemoryView(buf, self.format, itemsize)
+            mv = W_MemoryView.copy(self)
+            mv.slice(start, stop, step, size)
+            mv._init_flags()
+            return mv
         else:
-            # XXX needs to return a W_MemoryView with a NonContiguousSubBuffer
-            # maybe?  Need to check the cpyext requirements for that
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX extended slicing")
+            mv = W_MemoryView.copy(self)
+            mv.slice(start, stop, step, size)
+            mv.length = mv.bytecount_from_shape()
+            mv._init_flags()
+            return mv
+
+    def slice(self, start, stop, step, size):
+        # modifies the buffer, shape and stride to allow step to be > 1
+        # NOTE that start, stop & size are already byte offsets/count
+        # TODO subbuffer
+        strides = self.getstrides()[:]
+        shape = self.getshape()[:]
+        itemsize = self.getitemsize()
+        dim = 0
+        self.buf = SubBuffer(self.buf, strides[dim] * (start//itemsize), size*step)
+        shape[dim] = size
+        strides[dim] = strides[dim] * step
+        self.strides = strides
+        self.shape = shape
+
+    def bytecount_from_shape(self):
+        dim = self.getndim()
+        shape = self.getshape()
+        length = 1
+        for i in range(dim):
+            length *= shape[i]
+        return length * self.getitemsize()
+
+    @staticmethod
+    def copy(view, buf=None):
+        # TODO suboffsets
+        if buf == None:
+            buf = view.buf
+        return W_MemoryView(buf, view.getformat(), view.getitemsize(),
+                            view.getndim(), view.getshape()[:], view.getstrides()[:])
+
+    def _apply_itemsize(self, space, start, size, itemsize):
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+
+        stop  = start + size
+        # start & stop are now byte offset, thus use self.buf.getlength()
+        if stop > self.buf.getlength():
+            raise oefmt(space.w_IndexError, 'index out of range')
+
+        return start, stop, size
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         if space.isinstance_w(w_index, space.w_tuple):
-            raise oefmt(space.w_NotImplementedError, "XXX tuple setitem")
+            raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.itemsize
+        itemsize = self.getitemsize()
+        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
             if itemsize == 1:
                 ch = getbytevalue(space, w_obj)
@@ -125,16 +347,41 @@
                     raise oefmt(space.w_TypeError,
                                 "memoryview: invalid type for format '%s'",
                                 self.format)
-                self.buf.setslice(start * itemsize, fmtiter.result.build())
+                self.buf.setslice(start, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-            if value.getlength() != size * self.itemsize:
+            if value.getlength() != size:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
-            self.buf.setslice(start * itemsize, value.as_str())
+            self.buf.setslice(start, value.as_str())
         else:
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX extended slicing")
+            if self.getndim() != 1:
+                raise oefmt(space.w_NotImplementedError,
+                        "memoryview slice assignments are currently "
+                        "restricted to ndim = 1")
+            # this is the case of a one dimensional copy!
+            # NOTE we could maybe make use of copy_base, but currently we do not
+            itemsize = self.getitemsize()
+            data = []
+            src = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+            dst_strides = self.getstrides()
+            dim = 0
+            dst = SubBuffer(self.buf, start, size)
+            src_stride0 = dst_strides[dim]
+
+            off = 0
+            src_shape0 = size // itemsize
+            src_stride0 = src.getstrides()[0]
+            if isinstance(w_obj, W_MemoryView):
+                src_stride0 = w_obj.getstrides()[0]
+            for i in range(src_shape0):
+                data.append(src.getslice(off,off+itemsize,1,itemsize))
+                off += src_stride0
+            off = 0
+            dst_stride0 = self.getstrides()[0] * step
+            for dataslice in data:
+                dst.setslice(off, dataslice)
+                off += dst_stride0
 
     def descr_len(self, space):
         self._check_released(space)
@@ -142,11 +389,11 @@
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap(self.format)
+        return space.wrap(self.getformat())
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.newint(self.itemsize)
+        return space.wrap(self.itemsize)
 
     def w_get_ndim(self, space):
         self._check_released(space)
@@ -158,11 +405,11 @@
 
     def w_get_shape(self, space):
         self._check_released(space)
-        return space.newtuple([space.newint(self.getlength())])
+        return space.newtuple([space.wrap(x) for x in self.getshape()])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.newint(self.itemsize)])
+        return space.newtuple([space.wrap(x) for x in self.getstrides()])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)
@@ -240,20 +487,179 @@
             size = rffi.sizeof(rffi.VOIDP)
         return size
 
+    def _zero_in_shape(self):
+        # this method could be moved to the class Buffer
+        buf = self.buf
+        shape = buf.getshape()
+        for i in range(buf.getndim()):
+            if shape[i] == 0:
+                return True
+        return False
+
     def descr_cast(self, space, w_format, w_shape=None):
         self._check_released(space)
-        if not space.is_none(w_shape):
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX cast() with a shape")
+
+        if not space.isinstance_w(w_format, space.w_unicode):
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: format argument must be a string"))
+
         fmt = space.str_w(w_format)
-        newitemsize = self.get_native_fmtchar(fmt)
-        return W_MemoryView(self.buf, fmt, newitemsize)
+        buf = self.buf
+        ndim = 1
+
+        if not memory_view_c_contiguous(space, self.flags):
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: casts are restricted" \
+                               " to C-contiguous views"))
+
+        if (w_shape or buf.getndim() != 1) and self._zero_in_shape():
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: cannot casts view with" \
+                               " zeros in shape or strides"))
+
+        itemsize = self.get_native_fmtchar(fmt)
+        if w_shape:
+            if not (space.isinstance_w(w_shape, space.w_list) or space.isinstance_w(w_shape, space.w_tuple)):
+                raise oefmt(space.w_TypeError, "expected list or tuple got %T", w_shape)
+            ndim = space.len_w(w_shape)
+            if ndim > MEMORYVIEW_MAX_DIM:
+                raise oefmt(space.w_ValueError, \
+                        "memoryview: number of dimensions must not exceed %d",
+                        ndim)
+            # yes access ndim as field
+            if self.ndim > 1 and buf.getndim() != 1:
+                raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: cast must be 1D -> ND or ND -> 1D"))
+
+        mv = W_MemoryView(buf, self.format, self.itemsize)
+        origfmt = mv.getformat()
+        mv._cast_to_1D(space, origfmt, fmt, itemsize)
+        if w_shape:
+            fview = space.fixedview(w_shape)
+            shape = [space.int_w(w_obj) for w_obj in fview]
+            mv._cast_to_ND(space, shape, ndim)
+        return mv
+
+    def _init_flags(self):
+        buf = self.buf
+        ndim = buf.getndim()
+        flags = 0
+        if ndim == 0:
+            flags |= MEMORYVIEW_SCALAR | MEMORYVIEW_C | MEMORYVIEW_FORTRAN
+        if ndim == 1:
+            shape = buf.getshape()
+            strides = buf.getstrides()
+            if len(shape) > 0 and shape[0] == 1 and \
+               len(strides) > 0 and strides[0] == buf.getitemsize():
+                flags |= MEMORYVIEW_C | MEMORYVIEW_SCALAR
+        # TODO for now?
+        flags |= MEMORYVIEW_C
+        # TODO if buf.is_contiguous('C'):
+        # TODO     flags |= MEMORYVIEW_C
+        # TODO elif buf.is_contiguous('F'):
+        # TODO     flags |= MEMORYVIEW_FORTRAN
+
+        # TODO missing suboffsets
+
+        self.flags = flags
+
+    def _cast_to_1D(self, space, origfmt, fmt, itemsize):
+        buf = self.buf
+        if itemsize < 0:
+            raise oefmt(space.w_ValueError, "memoryview: destination" \
+                    " format must be a native single character format prefixed" \
+                    " with an optional '@'")
+
+        if self.get_native_fmtchar(origfmt) < 0 or \
+           (not is_byte_format(fmt) and not is_byte_format(origfmt)):
+            raise oefmt(space.w_TypeError,
+                    "memoryview: cannot cast between" \
+                    " two non-byte formats")
+
+        if buf.getlength() % itemsize != 0:
+            raise oefmt(space.w_TypeError,
+                    "memoryview: length is not a multiple of itemsize")
+
+        newfmt = self.get_native_fmtstr(fmt)
+        if not newfmt:
+            raise oefmt(space.w_RuntimeError,
+                    "memoryview: internal error")
+        self.format = newfmt
+        self.itemsize = itemsize
+        self.ndim = 1
+        self.shape = [buf.getlength() // itemsize]
+        self.strides = [itemsize]
+        # XX suboffsets
+
+        self._init_flags()
+
+    def get_native_fmtstr(self, fmt):
+        lenfmt = len(fmt)
+        nat = False
+        if lenfmt == 0:
+            return None
+        elif lenfmt == 1:
+            format = fmt[0] # fine!
+        elif lenfmt == 2:
+            if fmt[0] == '@':
+                nat = True
+                format = fmt[1]
+            else:
+                return None
+        else:
+            return None
+
+        chars = ['c','b','B','h','H','i','I','l','L','q',
+                 'Q','n','N','f','d','?','P']
+        for c in unrolling_iterable(chars):
+            if c == format:
+                if nat: return '@'+c
+                else: return c
+
+        return None
+
+    def _cast_to_ND(self, space, shape, ndim):
+        buf = self.buf
+
+        self.ndim = ndim
+        length = self.itemsize
+        if ndim == 0:
+            self.shape = []
+            self.strides = []
+        else:
+            self.shape = shape
+            for i in range(ndim):
+                length *= shape[i]
+            self._init_strides_from_shape()
+
+        if length != self.buf.getlength():
+            raise OperationError(space.w_TypeError,
+                    space.wrap("memoryview: product(shape) * itemsize != buffer size"))
+
+        self._init_flags()
+
+    def _init_strides_from_shape(self):
+        shape = self.getshape()
+        s = [0] * len(shape)
+        self.strides = s
+        ndim = self.getndim()
+        s[ndim-1] = self.itemsize
+        i = ndim-2
+        while i >= 0:
+            s[i] = s[i+1] * shape[i+1]
+            i -= 1
 
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         self._check_released(space)
         return _array_to_hexstring(space, self.buf)
 
+def is_byte_format(char):
+    return char == 'b' or char == 'B' or char == 'c'
+
+def memory_view_c_contiguous(space, flags):
+    return flags & (space.BUF_CONTIG_RO|MEMORYVIEW_C) != 0
+
 W_MemoryView.typedef = TypeDef(
     "memoryview",
     __doc__ = """\
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -1,3 +1,11 @@
+import py
+import struct
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.typedef import TypeDef
+from rpython.rlib.buffer import Buffer
+from pypy.conftest import option
+
 class AppTestMemoryView:
     spaceconfig = dict(usemodules=['array'])
 
@@ -38,7 +46,7 @@
         assert len(w) == 1
         assert list(w) == [97]
         v[::2] = b'ABC'
-        assert data == bytearray(b'AbBeCg')
+        assert data == bytearray(eval("b'AbBeCg'"))
 
     def test_memoryview_attrs(self):
         v = memoryview(b"a"*100)
@@ -217,4 +225,159 @@
         data = bytearray(b'abcdefghij')
         m3 = memoryview(data).cast('h')
         m3[1:5:2] = memoryview(b"xyXY").cast('h')
-        assert data == bytearray(b'abxyefXYij')
+        assert data == bytearray(eval("b'abxyefXYij'"))
+
+class MockBuffer(Buffer):
+    def __init__(self, space, w_arr, w_dim, w_fmt, \
+                 w_itemsize, w_strides, w_shape):
+        self.space = space
+        self.w_arr = w_arr
+        self.arr = []
+        self.ndim = space.int_w(w_dim)
+        self.format = space.str_w(w_fmt)
+        self.itemsize = space.int_w(w_itemsize)
+        self.strides = []
+        for w_i in w_strides.getitems_unroll():
+            self.strides.append(space.int_w(w_i))
+        self.shape = []
+        for w_i in w_shape.getitems_unroll():
+            self.shape.append(space.int_w(w_i))
+        self.readonly = True
+        self.shape.append(space.len_w(w_arr))
+        self.data = []
+        itemsize = 1
+        worklist = [(1,w_arr)]
+        while worklist:
+            dim, w_work = worklist.pop()
+            if space.isinstance_w(w_work, space.w_list):
+                for j, w_obj in enumerate(w_work.getitems_unroll()):
+                    worklist.insert(0, (dim+1, w_obj))
+                continue
+            byte = struct.pack(self.format, space.int_w(w_work))
+            for c in byte:
+                self.data.append(c)
+        self.data = ''.join(self.data)
+
+    def getformat(self):
+        return self.format
+
+    def getitem(self, index):
+        return self.data[index:index+1]
+
+    def getlength(self):
+        return len(self.data)
+
+    def getitemsize(self):
+        return self.itemsize
+
+    def getndim(self):
+        return self.ndim
+
+    def getstrides(self):
+        return self.strides
+
+    def getshape(self):
+        return self.shape
+
+    def is_contiguous(self, format):
+        return format == 'C'
+
+class W_MockArray(W_Root):
+    def __init__(self, w_list, w_dim, w_fmt, w_size, w_strides, w_shape):
+        self.w_list = w_list
+        self.w_dim = w_dim
+        self.w_fmt = w_fmt
+        self.w_size = w_size
+        self.w_strides = w_strides
+        self.w_shape = w_shape
+
+    @staticmethod
+    def descr_new(space, w_type, w_list, w_dim, w_fmt, \
+                         w_size, w_strides, w_shape):
+        return W_MockArray(w_list, w_dim, w_fmt, w_size, w_strides, w_shape)
+
+    def buffer_w(self, space, flags):
+        return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
+                          self.w_size, self.w_strides, self.w_shape)
+
+    def buffer_w_ex(self, space, flags):
+        return self.buffer_w(space, flags), space.str_w(self.w_fmt), space.int_w(self.w_size)
+
+W_MockArray.typedef = TypeDef("MockArray",
+    __new__ = interp2app(W_MockArray.descr_new),
+)
+
+class AppTestMemoryViewMockBuffer(object):
+    spaceconfig = dict(usemodules=[])
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("Impossible to run on appdirect")
+        cls.w_MockArray = cls.space.gettypefor(W_MockArray)
+
+    def test_tuple_indexing(self):
+        content = self.MockArray([[0,1,2,3], [4,5,6,7], [8,9,10,11]],
+                                 dim=2, fmt='B', size=1,
+                                 strides=[4,1], shape=[3,4])
+        view = memoryview(content)
+        assert view[0,0] == 0
+        assert view[2,0] == 8
+        assert view[2,3] == 11
+        assert view[-1,-1] == 11
+        assert view[-3,-4] == 0
+
+        raises(IndexError, "view.__getitem__((2**63-1,0))")
+        raises(TypeError, "view.__getitem__((0, 0, 0))")
+
+    def test_tuple_indexing_int(self):
+        content = self.MockArray([ [[1],[2],[3]], [[4],[5],[6]] ],
+                                 dim=3, fmt='i', size=4,
+                                 strides=[12,4,4], shape=[2,3,1])
+        view = memoryview(content)
+        assert view[0,0,0] == 1
+        assert view[-1,2,0] == 6
+
+    def test_cast_non_byte(self):
+        empty = self.MockArray([], dim=1, fmt='i', size=4, strides=[1], shape=[1])
+        view = memoryview(empty)
+        raises(TypeError, "view.cast('l')")
+        try:
+            view.cast('l')
+            assert False, "i -> l not possible. buffer must be byte format"
+        except TypeError:
+            pass
+
+    def test_cast_empty(self):
+        empty = self.MockArray([], dim=1, fmt='b', size=1, strides=[1], shape=[1])
+        view = memoryview(empty)
+        cview = view.cast('i')
+        assert cview.tobytes() == b''
+        assert cview.tolist() == []
+        assert view.format == 'b'
+        assert cview.format == 'i'
+        #
+        assert cview.cast('b').cast('q').cast('b').tolist() == []
+        #
+        assert cview.format == 'i'
+        raises(TypeError, "cview.cast('i')")
+
+    def test_cast_with_shape(self):
+        empty = self.MockArray([1,0,2,0,3,0],
+                    dim=1, fmt='h', size=2,
+                    strides=[8], shape=[6])
+        view = memoryview(empty)
+        byteview = view.cast('b')
+        assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]
+        i32view = byteview.cast('i', shape=[1,3])
+        assert i32view.format == 'i'
+        assert i32view.itemsize == 4
+        assert i32view.tolist() == [[1,2,3]]
+        i32view = byteview.cast('i', shape=(1,3))
+        assert i32view.tolist() == [[1,2,3]]
+
+    def test_cast_bytes(self):
+        bytes = b"\x02\x00\x03\x00\x04\x00" \
+                b"\x05\x00\x06\x00\x07\x00"
+        view = memoryview(bytes)
+        v = view.cast('h', shape=(3,2))
+        assert v.tolist() == [[2,3],[4,5],[6,7]]
+        raises(TypeError, "view.cast('h', shape=(3,3))")

From pypy.commits at gmail.com  Tue Aug 30 14:49:04 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 11:49:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <57c5d520.c70a1c0a.e04ff.cfc9@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86755:0673c426d243
Date: 2016-08-30 20:48 +0200
http://bitbucket.org/pypy/pypy/changeset/0673c426d243/

Log:	merge heads


From pypy.commits at gmail.com  Tue Aug 30 14:49:00 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 11:49:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-memoryview: translation fixes
Message-ID: <57c5d51c.d41a1c0a.3b852.0b3a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r86753:8a93b321520c
Date: 2016-08-30 20:38 +0200
http://bitbucket.org/pypy/pypy/changeset/8a93b321520c/

Log:	translation fixes

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -121,11 +121,13 @@
     def _copy_rec(self, idim, data, off):
         shapes = self.getshape()
         shape = shapes[idim]
+        strides = self.getstrides()
 
         if self.getndim()-1 == idim:
             self._copy_base(data,off)
             return
 
+        # TODO add a test that has at least 2 dims
         for i in range(shape):
             self._copy_rec(idim+1,data,off)
             off += strides[idim]

From pypy.commits at gmail.com  Tue Aug 30 14:51:30 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 30 Aug 2016 11:51:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: format(object(),
 'x') now raises an error (since 3.4)
Message-ID: <57c5d5b2.d4e01c0a.e9e5b.d285@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r86756:c1d25a7b9783
Date: 2016-08-30 19:50 +0100
http://bitbucket.org/pypy/pypy/changeset/c1d25a7b9783/

Log:	format(object(), 'x') now raises an error (since 3.4)

diff --git a/pypy/module/__builtin__/test/test_format.py b/pypy/module/__builtin__/test/test_format.py
--- a/pypy/module/__builtin__/test/test_format.py
+++ b/pypy/module/__builtin__/test/test_format.py
@@ -1,22 +1,7 @@
-class AppTestFormat:
+class AppTestFormat(object):
 
     def test_format(self):
-        """Test deprecation warnings from format(object(), 'nonempty')"""
-
-        import warnings
-
-        def test_deprecated(obj, fmt_str, should_raise_warning):
-            with warnings.catch_warnings(record=True) as w:
-                warnings.simplefilter("always", DeprecationWarning)
-                format(obj, fmt_str)
-            if should_raise_warning:
-                assert len(w) == 1
-                assert isinstance(w[0].message, DeprecationWarning)
-                assert 'object.__format__ with a non-empty format string '\
-                        in str(w[0].message)
-            else:
-                assert len(w) == 0
-
+        """Test error from format(object(), 'nonempty')"""
         fmt_strs = ['', 's']
 
         class A:
@@ -24,7 +9,7 @@
                 return format('', fmt_str)
 
         for fmt_str in fmt_strs:
-            test_deprecated(A(), fmt_str, False)
+            format(A(), fmt_str)  # does not raise
 
         class B:
             pass
@@ -34,5 +19,7 @@
 
         for cls in [object, B, C]:
             for fmt_str in fmt_strs:
-                print(cls, fmt_str)
-                test_deprecated(cls(), fmt_str, len(fmt_str) != 0)
+                if fmt_str:
+                    raises(TypeError, format, cls(), fmt_str)
+                else:
+                    format(cls(), fmt_str)  # does not raise
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -195,8 +195,8 @@
     else:
         raise oefmt(space.w_TypeError, "format_spec must be a string")
     if space.len_w(w_format_spec) > 0:
-        msg = "object.__format__ with a non-empty format string is deprecated"
-        space.warn(space.wrap(msg), space.w_DeprecationWarning)
+        raise oefmt(space.w_TypeError,
+                    "non-empty format string passed to object.__format__")
     return space.format(w_as_str, w_format_spec)
 
 def descr__eq__(space, w_self, w_other):

From pypy.commits at gmail.com  Tue Aug 30 15:35:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 Aug 2016 12:35:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix test
Message-ID: <57c5e01f.041f1c0a.8d1b3.dc5d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86757:63e420d9e2ff
Date: 2016-08-30 21:35 +0200
http://bitbucket.org/pypy/pypy/changeset/63e420d9e2ff/

Log:	Fix test

diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -125,8 +125,8 @@
     def test_optional(self):
         mod = self.get_ast("x(32)", "eval")
         call = mod.body
-        assert call.starargs is None
-        assert call.kwargs is None
+        assert len(call.args) == 1
+        assert call.args[0].n == 32
         co = compile(mod, "<test>", "eval")
         ns = {"x" : lambda x: x}
         assert eval(co, ns) == 32

From pypy.commits at gmail.com  Tue Aug 30 22:08:06 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 Aug 2016 19:08:06 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: add missing attribute
Message-ID: <57c63c06.08d11c0a.11d33.4374@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86758:f4f5cd8cb121
Date: 2016-08-31 05:05 +0300
http://bitbucket.org/pypy/pypy/changeset/f4f5cd8cb121/

Log:	add missing attribute

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,6 +335,9 @@
     def getshape(self):
         return self.shape
 
+    def getstrides(self):
+        return self.strides
+
     def getitemsize(self):
         return self.itemsize
 

From pypy.commits at gmail.com  Tue Aug 30 22:08:08 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 Aug 2016 19:08:08 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: test,
 implement creation of numpypy array from memoryview
Message-ID: <57c63c08.0205c20a.f2090.0133@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86759:844b80dca877
Date: 2016-08-31 05:07 +0300
http://bitbucket.org/pypy/pypy/changeset/844b80dca877/

Log:	test, implement creation of numpypy array from memoryview

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -465,6 +465,13 @@
         self.storage = storage
         self.start = start
         self.gcstruct = V_OBJECTSTORE
+        if dtype.num == NPY.OBJECT:
+            self.gcstruct = _create_objectstore(storage, self.size,
+                                                dtype.elsize)
+
+    def __del__(self):
+        if self.gcstruct:
+            self.gcstruct.length = 0
 
     def fill(self, space, box):
         self.dtype.itemtype.fill(
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -11,6 +11,7 @@
 from pypy.module.micronumpy.converters import shape_converter, order_converter
 import pypy.module.micronumpy.constants as NPY
 from .casting import scalar2dtype
+from pypy.objspace.std.memoryobject import W_MemoryView
 
 
 def build_scalar(space, w_dtype, w_state):
@@ -83,15 +84,13 @@
                     )
         if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)):
             data_w = space.listview(w_data)
-            data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
+            w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
             read_only = True # XXX why not space.is_true(data_w[1])
             offset = 0
-            return W_NDimArray.from_shape_and_storage(space, shape, data, 
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
                                     dtype, strides=strides, start=offset), read_only
         if w_data is None:
-            data = w_object
-        else:
-            data = w_data
+            w_data = w_object
         w_offset = space.finditem(w_interface, space.wrap('offset'))
         if w_offset is None:
             offset = 0
@@ -101,7 +100,7 @@
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
                    "__array_interface__ strides not fully supported yet") 
-        arr = frombuffer(space, data, dtype, support.product(shape), offset)
+        arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
         
@@ -110,6 +109,62 @@
             return None, False
         raise
 
+def _descriptor_from_pep3118_format(space, c_format):
+    descr = descriptor.decode_w_dtype(space, space.wrap(c_format))
+    if descr:
+        return descr
+    msg = "invalid PEP 3118 format string: '%s'" % c_format
+    space.warn(space.wrap(msg), space.w_RuntimeWarning)
+    return None 
+
+def _array_from_buffer_3118(space, w_object, dtype):
+    buf = w_object.buf
+    if buf.getformat():
+        descr = _descriptor_from_pep3118_format(space, buf.getformat())
+        if not descr:
+            return w_object
+        if dtype and descr:
+            raise oefmt(space.w_NotImplementedError,
+                "creating an array from a memoryview while specifying dtype "
+                "not supported")
+        if descr.elsize != buf.getitemsize(): 
+            msg = ("Item size computed from the PEP 3118 buffer format "
+                  "string does not match the actual item size.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+        dtype = descr 
+    elif not dtype:
+        dtype = descriptor.get_dtype_cache(space).w_stringdtype
+        dtype.elsize = buf.getitemsize()
+    nd = buf.getndim()
+    shape = buf.getshape()
+    strides = []
+    if shape:
+        strides = buf.getstrides()
+        if not strides:
+            d = len(buf)
+            strides = [0] * nd
+            for k in range(nd):
+                if shape[k] > 0:
+                    d /= shape[k]
+                    strides[k] = d
+    else:
+        if nd == 1:
+            shape = [len(buf) / dtype.elsize, ]
+            strides = [dtype.elsize, ]
+        elif nd > 1:
+            msg = ("ndim computed from the PEP 3118 buffer format "
+                   "is greater than 1, but shape is NULL.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+    storage = buf.get_raw_address()
+    writable = not buf.readonly
+    w_ret = W_NDimArray.from_shape_and_storage(space, shape, storage,
+               storage_bytes=len(buf), dtype=dtype, w_base=w_object, 
+               writable=writable, strides=strides)
+    if w_ret:
+        return w_ret
+    return w_object
 
 @unwrap_spec(ndmin=int, copy=bool, subok=bool)
 def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False,
@@ -139,6 +194,9 @@
             w_object = w_array
             copy = False
             dtype = w_object.get_dtype()
+        elif isinstance(w_object, W_MemoryView):
+            # use buffer interface
+            w_object = _array_from_buffer_3118(space, w_object, dtype)
     if not isinstance(w_object, W_NDimArray):
         w_array, _copy = try_interface_method(space, w_object)
         if w_array is not None:
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3594,6 +3594,7 @@
         cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
         cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
         cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+        cls.w_one = cls.space.wrap(struct.pack('i', 1))
 
     def test_frombuffer(self):
         import numpy as np
@@ -3645,8 +3646,6 @@
         else:
             EMPTY = None
         x = np.array([1, 2, 3, 4, 5], dtype='i')
-        y = memoryview('abc')
-        assert y.format == 'B'
         y = memoryview(x)
         assert y.format == 'i'
         assert y.shape == (5,)
@@ -3654,6 +3653,16 @@
         assert y.strides == (4,)
         assert y.suboffsets == EMPTY
         assert y.itemsize == 4
+        assert isinstance(y, memoryview)
+        assert y[0] == self.one
+        assert (np.array(y) == x).all()
+
+        x = np.array([0, 0, 0, 0], dtype='O')
+        y = memoryview(x)
+        # handles conversion of address to pinned object?
+        z = np.array(y)
+        assert z.dtype == 'O'
+        assert (z == x).all()
 
     def test_fromstring(self):
         import sys

From pypy.commits at gmail.com  Wed Aug 31 01:52:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 22:52:51 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: add buffer interface functions
 to ArrayBuffer (python array module)
Message-ID: <57c670b3.2472c20a.fa9b1.4645@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: buffer-interface
Changeset: r86760:bd907242e194
Date: 2016-08-31 07:52 +0200
http://bitbucket.org/pypy/pypy/changeset/bd907242e194/

Log:	add buffer interface functions to ArrayBuffer (python array module)

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -597,6 +597,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()

From pypy.commits at gmail.com  Wed Aug 31 02:56:40 2016
From: pypy.commits at gmail.com (stefanor)
Date: Tue, 30 Aug 2016 23:56:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: Make customize_compiler idempotent
Message-ID: <57c67fa8.a6a5c20a.b957f.496e@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r86761:a95e07807311
Date: 2016-08-30 23:55 -0700
http://bitbucket.org/pypy/pypy/changeset/a95e07807311/

Log:	Make customize_compiler idempotent

	Otherwise, it modifies the mutable list on the class, which leads to
	later unhappiness

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (

From pypy.commits at gmail.com  Wed Aug 31 02:59:06 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 30 Aug 2016 23:59:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-byteformat: add __mod__ to W_BytesObject,
 use mod_format to format the bytes object
Message-ID: <57c6803a.e97ac20a.936fc.4bd5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-byteformat
Changeset: r86762:e551d8bfa0da
Date: 2016-08-31 08:58 +0200
http://bitbucket.org/pypy/pypy/changeset/e551d8bfa0da/

Log:	add __mod__ to W_BytesObject, use mod_format to format the bytes
	object

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -14,6 +14,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
+from pypy.objspace.std.formatting import mod_format, FORMAT_BYTES
 
 
 class W_AbstractBytesObject(W_Root):
@@ -394,6 +395,12 @@
         of the specified width. The string S is never truncated.
         """
 
+    def descr_mod(self, space, w_values):
+        """S % values -> string
+
+        Format bytes objects
+        """
+
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
@@ -663,6 +670,9 @@
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         return _array_to_hexstring(space, StringBuffer(self._value))
 
+    def descr_mod(self, space, w_values):
+        return mod_format(space, self, w_values, fmt_type=FORMAT_BYTES)
+
     @staticmethod
     def _iter_getitem_result(self, space, index):
         assert isinstance(self, W_BytesObject)
@@ -803,6 +813,8 @@
     __mul__ = interpindirect2app(W_AbstractBytesObject.descr_mul),
     __rmul__ = interpindirect2app(W_AbstractBytesObject.descr_rmul),
 
+    __mod__ = interpindirect2app(W_AbstractBytesObject.descr_mod),
+
     __getitem__ = interpindirect2app(W_AbstractBytesObject.descr_getitem),
 
     capitalize = interpindirect2app(W_AbstractBytesObject.descr_capitalize),
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -494,9 +494,14 @@
     [_name[-1] for _name in dir(StringFormatter)
                if len(_name) == 5 and _name.startswith('fmt_')])
 
-def format(space, w_fmt, values_w, w_valuedict, do_unicode):
+FORMAT_STR = 0
+FORMAT_UNICODE = 1
+FORMAT_BYTES = 2
+FORMAT_BYTEARRAY = 3
+
+def format(space, w_fmt, values_w, w_valuedict, fmt_type):
     "Entry point"
-    if not do_unicode:
+    if fmt_type != FORMAT_UNICODE:
         fmt = space.str_w(w_fmt)
         formatter = StringFormatter(space, fmt, values_w, w_valuedict)
         try:
@@ -505,25 +510,29 @@
             # fall through to the unicode case
             pass
         else:
+            if fmt_type == FORMAT_BYTES:
+                return space.newbytes(result)
+            elif fmt_type == FORMAT_BYTEARRAY:
+                return space.newbytearray(result)
             return space.wrap(result)
     fmt = space.unicode_w(w_fmt)
     formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict)
     result = formatter.format()
     return space.wrap(result)
 
-def mod_format(space, w_format, w_values, do_unicode=False):
+def mod_format(space, w_format, w_values, fmt_type=FORMAT_STR):
     if space.isinstance_w(w_values, space.w_tuple):
         values_w = space.fixedview(w_values)
-        return format(space, w_format, values_w, None, do_unicode)
+        return format(space, w_format, values_w, None, fmt_type)
     else:
         # we check directly for dict to avoid obscure checking
         # in simplest case
         if space.isinstance_w(w_values, space.w_dict) or \
            (space.lookup(w_values, '__getitem__') and
            not space.isinstance_w(w_values, space.w_unicode)):
-            return format(space, w_format, [w_values], w_values, do_unicode)
+            return format(space, w_format, [w_values], w_values, fmt_type)
         else:
-            return format(space, w_format, [w_values], None, do_unicode)
+            return format(space, w_format, [w_values], None, fmt_type)
 
 # ____________________________________________________________
 # Formatting helpers
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -346,6 +346,9 @@
     def newbytes(self, s):
         return W_BytesObject(s)
 
+    def newbytearray(self, l):
+        return W_BytearrayObject(l)
+
     def newunicode(self, uni):
         return W_UnicodeObject(uni)
 
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -97,10 +97,6 @@
         assert bytes('abc', 'ascii') == b'abc'
         assert bytes(set(b'foo')) in (b'fo', b'of')
 
-    def test_format(self):
-        import operator
-        raises(TypeError, operator.mod, b"%s", (1,))
-
     def test_fromhex(self):
         assert bytes.fromhex("abcd") == b'\xab\xcd'
         assert b''.fromhex("abcd") == b'\xab\xcd'
@@ -877,3 +873,11 @@
                "73616e746120636c617573"
         assert bytes(64).hex() == "00"*64
 
+    def test_format(self):
+        assert eval("b'a%db' % 2") == eval("b'a2b'")
+        assert eval("b'00%.2f'").__mod__((0.01234)) == eval("b'000.01'")
+        assert eval("b'%04X' % 10") == eval("b'000A'")
+        assert eval("b'%c' % 48") == eval("b'0'")
+        assert eval("b'%c' % b'a'") == eval("b'a'")
+        assert eval("b'%b' % b'abc'") == eval("b'abc'")
+
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -15,7 +15,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.module.unicodedata import unicodedb
 from pypy.objspace.std import newformat
-from pypy.objspace.std.formatting import mod_format
+from pypy.objspace.std.formatting import mod_format, FORMAT_UNICODE
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
@@ -392,7 +392,7 @@
                                        self)
 
     def descr_mod(self, space, w_values):
-        return mod_format(space, self, w_values, do_unicode=True)
+        return mod_format(space, self, w_values, fmt_type=FORMAT_UNICODE)
 
     def descr_translate(self, space, w_table):
         selfvalue = self._value

From pypy.commits at gmail.com  Wed Aug 31 03:14:15 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 31 Aug 2016 00:14:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-byteformat: add more edges cases to the
 test as described in the pep 461
Message-ID: <57c683c7.0205c20a.f2090.4986@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-byteformat
Changeset: r86763:0fe90d1f2c7e
Date: 2016-08-31 09:13 +0200
http://bitbucket.org/pypy/pypy/changeset/0fe90d1f2c7e/

Log:	add more edges cases to the test as described in the pep 461

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -477,6 +477,9 @@
                                     "character code not in range(256)")
                     self.std_wp(s)
 
+        def fmt_b(self, w_value):
+            raise NotImplementedError
+
     return StringFormatter
 
 
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -874,10 +874,15 @@
         assert bytes(64).hex() == "00"*64
 
     def test_format(self):
-        assert eval("b'a%db' % 2") == eval("b'a2b'")
-        assert eval("b'00%.2f'").__mod__((0.01234)) == eval("b'000.01'")
-        assert eval("b'%04X' % 10") == eval("b'000A'")
-        assert eval("b'%c' % 48") == eval("b'0'")
-        assert eval("b'%c' % b'a'") == eval("b'a'")
-        assert eval("b'%b' % b'abc'") == eval("b'abc'")
+        """
+        assert b'a%db' % 2 == b'a2b'
+        assert b'00%.2f'.__mod__((0.01234,)) == b'000.01'
+        assert b'%04X' % 10 == b'000A'
+        assert b'%c' % 48 == b'0'
+        assert b'%c' % b'a' == b'a'
+        assert b'%b' % b'abc' == b'abc'
+        assert b'%b' % '&#12399;&#12356;'.encode('utf-8') == b'\xe3\x81\xaf\xe3\x81\x84'
+        raises(TypeError, 'b"%b" % 3.14')
+        raises(TypeError, 'b"%b" % "hello world"')
+        """
 

From pypy.commits at gmail.com  Wed Aug 31 03:25:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 31 Aug 2016 00:25:58 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: add me to pep 461 task (bytes %
 format)
Message-ID: <57c68686.d42f1c0a.8eefb.9640@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5694:566907ba0aec
Date: 2016-08-31 08:07 +0200
http://bitbucket.org/pypy/extradoc/changeset/566907ba0aec/

Log:	add me to pep 461 task (bytes % format)

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/2016-august-progress.rst
--- a/planning/py3.5/2016-august-progress.rst
+++ b/planning/py3.5/2016-august-progress.rst
@@ -4,13 +4,14 @@
 Planned
 -------
 
-* Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951 (plan_rich)
+* richard: Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951
   Seems to work, but test suite hangs to verify the CPython tests.
-* tuple indexing for memory view, !will be merged soon! (plan_rich)
+* richard: tuple indexing for memory view,
   Comments: Stronly tied to numpy. Hard to implement, because most of the basics are missing (dimensions/strides)
   We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
   Matti's opinion on that would be great!
-* plan_rich: extended slicing for memory view
+* richard: extended slicing for memory view
+* richard: bytes % args, bytearray % args: PEP 461
 
 * arigo: look at test failures relaced to os.scandir() or the pathlib
   module, or the enum module

From pypy.commits at gmail.com  Wed Aug 31 04:26:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 01:26:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2386: non-latin1 unicode keys
 were ignored in
Message-ID: <57c69499.45c8c20a.e6dc0.7ab0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86765:728a9942eb3e
Date: 2016-08-31 10:25 +0200
http://bitbucket.org/pypy/pypy/changeset/728a9942eb3e/

Log:	Issue #2386: non-latin1 unicode keys were ignored in
	'unicode.format(**d)'

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -8,6 +8,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rfloat import copysign, formatd
 from rpython.rlib.rarithmetic import r_uint, intmask
+from pypy.interpreter.signature import Signature
 
 
 @specialize.argtype(1)
@@ -40,6 +41,9 @@
 ANS_MANUAL = 3
 
 
+format_signature = Signature([], 'args', 'kwargs')
+
+
 def make_template_formatting_class(for_unicode):
     class TemplateFormatter(object):
         is_unicode = for_unicode
@@ -52,7 +56,17 @@
             self.template = template
 
         def build(self, args):
-            self.args, self.kwargs = args.unpack()
+            if self.is_unicode:
+                # for unicode, use the slower parse_obj() to get self.w_kwargs
+                # as a wrapped dictionary that may contain full-range unicode
+                # keys.  See test_non_latin1_key
+                space = self.space
+                w_args, w_kwds = args.parse_obj(None, 'format',
+                                                format_signature)
+                self.args = space.listview(w_args)
+                self.w_kwargs = w_kwds
+            else:
+                self.args, self.kwargs = args.unpack()
             self.auto_numbering = 0
             self.auto_numbering_state = ANS_INIT
             return self._build_string(0, len(self.template), 2)
@@ -197,17 +211,13 @@
             if index == -1:
                 kwarg = name[:i]
                 if self.is_unicode:
+                    w_arg = space.getitem(self.w_kwargs, space.wrap(kwarg))
+                else:
                     try:
-                        arg_key = kwarg.encode("latin-1")
-                    except UnicodeEncodeError:
-                        # Not going to be found in a dict of strings.
-                        raise OperationError(space.w_KeyError, space.wrap(kwarg))
-                else:
-                    arg_key = kwarg
-                try:
-                    w_arg = self.kwargs[arg_key]
-                except KeyError:
-                    raise OperationError(space.w_KeyError, space.wrap(arg_key))
+                        w_arg = self.kwargs[kwarg]
+                    except KeyError:
+                        raise OperationError(space.w_KeyError,
+                                             space.wrap(kwarg))
             else:
                 try:
                     w_arg = self.args[index]
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -215,7 +215,9 @@
         assert self.s("{!r}").format(x()) == self.s("32")
 
     def test_non_latin1_key(self):
-        raises(KeyError, self.s("{\u1000}").format)
+        raises(KeyError, u"{\u1000}".format)
+        d = {u"\u1000": u"foo"}
+        assert u"{\u1000}".format(**d) == u"foo"
 
 
 class AppTestStringFormat(BaseStringFormatTests):

From pypy.commits at gmail.com  Wed Aug 31 04:25:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 01:25:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Move 'self.is_unicode' to a class
 attribute (it's a constant anyway)
Message-ID: <57c69497.88cb1c0a.55ab1.a7a4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86764:e0f2d6ff08e9
Date: 2016-08-31 10:07 +0200
http://bitbucket.org/pypy/pypy/changeset/e0f2d6ff08e9/

Log:	Move 'self.is_unicode' to a class attribute (it's a constant anyway)

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -40,15 +40,15 @@
 ANS_MANUAL = 3
 
 
-def make_template_formatting_class():
+def make_template_formatting_class(for_unicode):
     class TemplateFormatter(object):
+        is_unicode = for_unicode
 
         parser_list_w = None
 
-        def __init__(self, space, is_unicode, template):
+        def __init__(self, space, template):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.template = template
 
         def build(self, args):
@@ -351,14 +351,8 @@
             return space.iter(space.newlist(self.parser_list_w))
     return TemplateFormatter
 
-StrTemplateFormatter = make_template_formatting_class()
-UnicodeTemplateFormatter = make_template_formatting_class()
-
-def str_template_formatter(space, template):
-    return StrTemplateFormatter(space, False, template)
-
-def unicode_template_formatter(space, template):
-    return UnicodeTemplateFormatter(space, True, template)
+str_template_formatter = make_template_formatting_class(for_unicode=False)
+unicode_template_formatter = make_template_formatting_class(for_unicode=True)
 
 
 def format_method(space, w_string, args, is_unicode):
@@ -395,16 +389,16 @@
 
 LONG_DIGITS = string.digits + string.ascii_lowercase
 
-def make_formatting_class():
+def make_formatting_class(for_unicode):
     class Formatter(BaseFormatter):
         """__format__ implementation for builtin types."""
 
+        is_unicode = for_unicode
         _grouped_digits = None
 
-        def __init__(self, space, is_unicode, spec):
+        def __init__(self, space, spec):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.spec = spec
 
         def _is_alignment(self, c):
@@ -1138,15 +1132,8 @@
             self._unknown_presentation("complex")
     return Formatter
 
-StrFormatter = make_formatting_class()
-UnicodeFormatter = make_formatting_class()
-
-
-def unicode_formatter(space, spec):
-    return StrFormatter(space, True, spec)
-
-def str_formatter(space, spec):
-    return UnicodeFormatter(space, False, spec)
+str_formatter = make_formatting_class(for_unicode=False)
+unicode_formatter = make_formatting_class(for_unicode=True)
 
 
 @specialize.arg(2)

From pypy.commits at gmail.com  Wed Aug 31 04:41:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 01:41:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c69820.11051c0a.a4c42.aec5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86766:c080e77ee085
Date: 2016-08-31 10:40 +0200
http://bitbucket.org/pypy/pypy/changeset/c080e77ee085/

Log:	hg merge default

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -27,6 +27,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -114,7 +122,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -303,6 +303,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -8,6 +8,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rfloat import copysign, formatd
 from rpython.rlib.rarithmetic import r_uint, intmask
+from pypy.interpreter.signature import Signature
 
 
 @specialize.argtype(1)
@@ -40,15 +41,18 @@
 ANS_MANUAL = 3
 
 
-def make_template_formatting_class():
+format_signature = Signature([], 'args', 'kwargs')
+
+
+def make_template_formatting_class(for_unicode):
     class TemplateFormatter(object):
+        is_unicode = for_unicode
 
         parser_list_w = None
 
-        def __init__(self, space, is_unicode, template):
+        def __init__(self, space, template):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.template = template
 
         def build(self, args, w_kwargs):
@@ -198,14 +202,10 @@
             if index == -1:
                 kwarg = name[:i]
                 if self.is_unicode:
-                    try:
-                        arg_key = kwarg.encode("latin-1")
-                    except UnicodeEncodeError:
-                        # Not going to be found in a dict of strings.
-                        raise OperationError(space.w_KeyError, space.wrap(kwarg))
+                    w_kwarg = space.newunicode(kwarg)
                 else:
-                    arg_key = kwarg
-                w_arg = space.getitem(self.w_kwargs, space.wrap(arg_key))
+                    w_kwarg = space.newbytes(kwarg)
+                w_arg = space.getitem(self.w_kwargs, w_kwarg)
             else:
                 if self.args is None:
                     w_msg = space.wrap("Format string contains positional "
@@ -356,14 +356,8 @@
             return space.iter(space.newlist(self.parser_list_w))
     return TemplateFormatter
 
-StrTemplateFormatter = make_template_formatting_class()
-UnicodeTemplateFormatter = make_template_formatting_class()
-
-def str_template_formatter(space, template):
-    return StrTemplateFormatter(space, False, template)
-
-def unicode_template_formatter(space, template):
-    return UnicodeTemplateFormatter(space, True, template)
+str_template_formatter = make_template_formatting_class(for_unicode=False)
+unicode_template_formatter = make_template_formatting_class(for_unicode=True)
 
 
 def format_method(space, w_string, args, w_kwargs, is_unicode):
@@ -400,16 +394,16 @@
 
 LONG_DIGITS = string.digits + string.ascii_lowercase
 
-def make_formatting_class():
+def make_formatting_class(for_unicode):
     class Formatter(BaseFormatter):
         """__format__ implementation for builtin types."""
 
+        is_unicode = for_unicode
         _grouped_digits = None
 
-        def __init__(self, space, is_unicode, spec):
+        def __init__(self, space, spec):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.spec = spec
 
         def _is_alignment(self, c):
@@ -1146,12 +1140,9 @@
             self._unknown_presentation("complex")
     return Formatter
 
-StrFormatter = make_formatting_class()
+unicode_formatter = make_formatting_class(for_unicode=True)
 
 
-def unicode_formatter(space, spec):
-    return StrFormatter(space, True, spec)
-
 @specialize.arg(2)
 def run_formatter(space, w_format_spec, meth, *args):
     formatter = unicode_formatter(space, space.unicode_w(w_format_spec))
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -157,6 +157,7 @@
                 # poor man's x.decode('ascii', 'replace'), since it's not
                 # supported by RPython
                 if not we_are_translated():
+                    import pdb;pdb.set_trace()
                     print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
                 lst = []
                 for ch in x:
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -216,7 +216,9 @@
         assert self.s("{!r}").format(x()) == "32"
 
     def test_non_latin1_key(self):
-        raises(KeyError, self.s("{\u1000}").format)
+        raises(KeyError, u"{\u1000}".format)
+        d = {u"\u1000": u"foo"}
+        assert u"{\u1000}".format(**d) == u"foo"
 
 
 class AppTestBoolFormat:
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -379,7 +379,11 @@
         w_kwds = space.newdict()
         if __args__.keywords:
             for i in range(len(__args__.keywords)):
-                space.setitem(w_kwds, space.wrap(__args__.keywords[i]),
+                try:     # pff
+                    arg = __args__.keywords[i].decode('utf-8')
+                except UnicodeDecodeError:
+                    continue   # uh, just skip that
+                space.setitem(w_kwds, space.newunicode(arg),
                               __args__.keywords_w[i])
         return newformat.format_method(space, self, __args__.arguments_w,
                                        w_kwds, True)
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -526,7 +526,7 @@
                  fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socket() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.
@@ -655,7 +655,7 @@
         address, addr_p, addrlen_p = self._addrbuf()
         try:
             remove_inheritable = not inheritable
-            if (not inheritable and SOCK_CLOEXEC is not None
+            if (not inheritable and 'SOCK_CLOEXEC' in constants
                     and _c.HAVE_ACCEPT4
                     and _accept4_syscall.attempt_syscall()):
                 newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
@@ -1138,7 +1138,7 @@
         try:
             res = -1
             remove_inheritable = not inheritable
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socketpair() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.

From pypy.commits at gmail.com  Wed Aug 31 04:41:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 01:41:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: merge heads
Message-ID: <57c69822.031dc20a.f5e91.810c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86767:8be5d923f6ae
Date: 2016-08-31 10:40 +0200
http://bitbucket.org/pypy/pypy/changeset/8be5d923f6ae/

Log:	merge heads

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -850,6 +850,9 @@
         raises(TypeError, delattr, A(), 42)
 
     def test_getattr_None(self):
+        import sys
+        if '__pypy__' not in sys.modules:
+            skip('CPython uses wrapper types for this')
         from types import FunctionType, MethodType
         assert isinstance(getattr(type(None), '__eq__'), FunctionType)
         assert isinstance(getattr(None, '__eq__'), MethodType)
@@ -864,4 +867,3 @@
         assert isinstance(getattr(a, '__eq__'), MethodType)
         a.__eq__ = 42
         assert a.__eq__ == 42
-

From pypy.commits at gmail.com  Wed Aug 31 04:45:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 01:45:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Move this loop out of the way of the JIT
Message-ID: <57c69934.4abf1c0a.79d0a.b669@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86768:60119aad4cd8
Date: 2016-08-31 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/60119aad4cd8/

Log:	Move this loop out of the way of the JIT

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -375,16 +375,19 @@
             raise
         return space.newbool(res)
 
+    def _parse_format_arg(self, space, w_kwds, __args__):
+        for i in range(len(__args__.keywords)):
+            try:     # pff
+                arg = __args__.keywords[i].decode('utf-8')
+            except UnicodeDecodeError:
+                continue   # uh, just skip that
+            space.setitem(w_kwds, space.newunicode(arg),
+                          __args__.keywords_w[i])
+
     def descr_format(self, space, __args__):
         w_kwds = space.newdict()
         if __args__.keywords:
-            for i in range(len(__args__.keywords)):
-                try:     # pff
-                    arg = __args__.keywords[i].decode('utf-8')
-                except UnicodeDecodeError:
-                    continue   # uh, just skip that
-                space.setitem(w_kwds, space.newunicode(arg),
-                              __args__.keywords_w[i])
+            self._parse_format_arg(space, w_kwds, __args__)
         return newformat.format_method(space, self, __args__.arguments_w,
                                        w_kwds, True)
 

From pypy.commits at gmail.com  Wed Aug 31 04:45:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 01:45:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c69945.45c8c20a.e6dc0.8268@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86769:37ffd1a52ce6
Date: 2016-08-31 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/37ffd1a52ce6/

Log:	hg merge py3k

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -27,6 +27,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -114,7 +122,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -850,6 +850,9 @@
         raises(TypeError, delattr, A(), 42)
 
     def test_getattr_None(self):
+        import sys
+        if '__pypy__' not in sys.modules:
+            skip('CPython uses wrapper types for this')
         from types import FunctionType, MethodType
         assert isinstance(getattr(type(None), '__eq__'), FunctionType)
         assert isinstance(getattr(None, '__eq__'), MethodType)
@@ -864,4 +867,3 @@
         assert isinstance(getattr(a, '__eq__'), MethodType)
         a.__eq__ = 42
         assert a.__eq__ == 42
-
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -303,6 +303,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -8,6 +8,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rfloat import copysign, formatd
 from rpython.rlib.rarithmetic import r_uint, intmask
+from pypy.interpreter.signature import Signature
 
 
 @specialize.argtype(1)
@@ -40,15 +41,18 @@
 ANS_MANUAL = 3
 
 
-def make_template_formatting_class():
+format_signature = Signature([], 'args', 'kwargs')
+
+
+def make_template_formatting_class(for_unicode):
     class TemplateFormatter(object):
+        is_unicode = for_unicode
 
         parser_list_w = None
 
-        def __init__(self, space, is_unicode, template):
+        def __init__(self, space, template):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.template = template
 
         def build(self, args, w_kwargs):
@@ -198,14 +202,10 @@
             if index == -1:
                 kwarg = name[:i]
                 if self.is_unicode:
-                    try:
-                        arg_key = kwarg.encode("latin-1")
-                    except UnicodeEncodeError:
-                        # Not going to be found in a dict of strings.
-                        raise OperationError(space.w_KeyError, space.wrap(kwarg))
+                    w_kwarg = space.newunicode(kwarg)
                 else:
-                    arg_key = kwarg
-                w_arg = space.getitem(self.w_kwargs, space.wrap(arg_key))
+                    w_kwarg = space.newbytes(kwarg)
+                w_arg = space.getitem(self.w_kwargs, w_kwarg)
             else:
                 if self.args is None:
                     w_msg = space.wrap("Format string contains positional "
@@ -356,14 +356,8 @@
             return space.iter(space.newlist(self.parser_list_w))
     return TemplateFormatter
 
-StrTemplateFormatter = make_template_formatting_class()
-UnicodeTemplateFormatter = make_template_formatting_class()
-
-def str_template_formatter(space, template):
-    return StrTemplateFormatter(space, False, template)
-
-def unicode_template_formatter(space, template):
-    return UnicodeTemplateFormatter(space, True, template)
+str_template_formatter = make_template_formatting_class(for_unicode=False)
+unicode_template_formatter = make_template_formatting_class(for_unicode=True)
 
 
 def format_method(space, w_string, args, w_kwargs, is_unicode):
@@ -400,16 +394,16 @@
 
 LONG_DIGITS = string.digits + string.ascii_lowercase
 
-def make_formatting_class():
+def make_formatting_class(for_unicode):
     class Formatter(BaseFormatter):
         """__format__ implementation for builtin types."""
 
+        is_unicode = for_unicode
         _grouped_digits = None
 
-        def __init__(self, space, is_unicode, spec):
+        def __init__(self, space, spec):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.spec = spec
 
         def _is_alignment(self, c):
@@ -1146,12 +1140,9 @@
             self._unknown_presentation("complex")
     return Formatter
 
-StrFormatter = make_formatting_class()
+unicode_formatter = make_formatting_class(for_unicode=True)
 
 
-def unicode_formatter(space, spec):
-    return StrFormatter(space, True, spec)
-
 @specialize.arg(2)
 def run_formatter(space, w_format_spec, meth, *args):
     formatter = unicode_formatter(space, space.unicode_w(w_format_spec))
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -157,6 +157,7 @@
                 # poor man's x.decode('ascii', 'replace'), since it's not
                 # supported by RPython
                 if not we_are_translated():
+                    import pdb;pdb.set_trace()
                     print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
                 lst = []
                 for ch in x:
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -216,7 +216,9 @@
         assert self.s("{!r}").format(x()) == "32"
 
     def test_non_latin1_key(self):
-        raises(KeyError, self.s("{\u1000}").format)
+        raises(KeyError, u"{\u1000}".format)
+        d = {u"\u1000": u"foo"}
+        assert u"{\u1000}".format(**d) == u"foo"
 
 
 class AppTestBoolFormat:
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -375,12 +375,19 @@
             raise
         return space.newbool(res)
 
+    def _parse_format_arg(self, space, w_kwds, __args__):
+        for i in range(len(__args__.keywords)):
+            try:     # pff
+                arg = __args__.keywords[i].decode('utf-8')
+            except UnicodeDecodeError:
+                continue   # uh, just skip that
+            space.setitem(w_kwds, space.newunicode(arg),
+                          __args__.keywords_w[i])
+
     def descr_format(self, space, __args__):
         w_kwds = space.newdict()
         if __args__.keywords:
-            for i in range(len(__args__.keywords)):
-                space.setitem(w_kwds, space.wrap(__args__.keywords[i]),
-                              __args__.keywords_w[i])
+            self._parse_format_arg(space, w_kwds, __args__)
         return newformat.format_method(space, self, __args__.arguments_w,
                                        w_kwds, True)
 
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -526,7 +526,7 @@
                  fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socket() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.
@@ -655,7 +655,7 @@
         address, addr_p, addrlen_p = self._addrbuf()
         try:
             remove_inheritable = not inheritable
-            if (not inheritable and SOCK_CLOEXEC is not None
+            if (not inheritable and 'SOCK_CLOEXEC' in constants
                     and _c.HAVE_ACCEPT4
                     and _accept4_syscall.attempt_syscall()):
                 newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
@@ -1138,7 +1138,7 @@
         try:
             res = -1
             remove_inheritable = not inheritable
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socketpair() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.

From pypy.commits at gmail.com  Wed Aug 31 04:58:52 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 31 Aug 2016 01:58:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-byteformat: redirect format r to a in py3
Message-ID: <57c69c4c.c4ebc20a.62fc2.739f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-byteformat
Changeset: r86770:8007dd7e86a6
Date: 2016-08-31 10:15 +0200
http://bitbucket.org/pypy/pypy/changeset/8007dd7e86a6/

Log:	redirect format r to a in py3

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -438,7 +438,7 @@
             self.std_wp(s)
 
         def fmt_r(self, w_value):
-            self.std_wp(self.space.unicode_w(self.space.repr(w_value)))
+            self.fmt_a(w_value)
 
         def fmt_a(self, w_value):
             from pypy.objspace.std.unicodeobject import ascii_from_object

From pypy.commits at gmail.com  Wed Aug 31 04:58:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 31 Aug 2016 01:58:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-byteformat: add test and impl %b format
Message-ID: <57c69c4e.c4ebc20a.62fc2.73a4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-byteformat
Changeset: r86771:45a8ee6fd44d
Date: 2016-08-31 10:58 +0200
http://bitbucket.org/pypy/pypy/changeset/45a8ee6fd44d/

Log:	add test and impl %b format

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -478,7 +478,27 @@
                     self.std_wp(s)
 
         def fmt_b(self, w_value):
-            raise NotImplementedError
+            space = self.space
+            # cpython explicitly checks for bytes & bytearray
+            if space.isinstance_w(w_value, space.w_bytes):
+                self.std_wp(space.bytes_w(w_value))
+                return
+            if space.isinstance_w(w_value, space.w_bytearray):
+                self.std_wp(space.bytes_w(w_value))
+                return
+
+            w_bytes_method = space.lookup(w_value, "__bytes__")
+            if w_bytes_method is not None:
+                w_bytes = space.get_and_call_function(w_bytes_method, w_value)
+                if not space.isinstance_w(w_bytes, space.w_bytes):
+                    raise oefmt(space.w_TypeError,
+                                "__bytes__ returned non-bytes (type '%T')", w_bytes)
+                self.std_wp(space.bytes_w(w_bytes))
+                return
+
+            raise oefmt(space.w_TypeError,
+                    "requires bytes, or an object that" \
+                    "implements __bytes__, not '%T'", w_value)
 
     return StringFormatter
 
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 class TestW_BytesObject:
 
     def teardown_method(self, method):
@@ -880,8 +881,12 @@
         assert b'%04X' % 10 == b'000A'
         assert b'%c' % 48 == b'0'
         assert b'%c' % b'a' == b'a'
+        """
+
+    def test_format_b(self):
+        """
         assert b'%b' % b'abc' == b'abc'
-        assert b'%b' % '&#12399;&#12356;'.encode('utf-8') == b'\xe3\x81\xaf\xe3\x81\x84'
+        assert b'%b' % u'&#12399;&#12356;'.encode('utf-8') == u'&#12399;&#12356;'.encode('utf-8')
         raises(TypeError, 'b"%b" % 3.14')
         raises(TypeError, 'b"%b" % "hello world"')
         """

From pypy.commits at gmail.com  Wed Aug 31 05:00:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 02:00:26 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Rename the file
Message-ID: <57c69caa.c19d1c0a.11b09.bb9f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5695:1ef0806eb0e8
Date: 2016-08-31 11:00 +0200
http://bitbucket.org/pypy/extradoc/changeset/1ef0806eb0e8/

Log:	Rename the file

diff --git a/planning/py3.5/2016-august-progress.rst b/planning/py3.5/milestone-1-progress.rst
rename from planning/py3.5/2016-august-progress.rst
rename to planning/py3.5/milestone-1-progress.rst

From pypy.commits at gmail.com  Wed Aug 31 05:13:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 02:13:39 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Rename the sections
Message-ID: <57c69fc3.c4ebc20a.62fc2.7acf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5696:55019142e241
Date: 2016-08-31 11:13 +0200
http://bitbucket.org/pypy/extradoc/changeset/55019142e241/

Log:	Rename the sections

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -1,8 +1,9 @@
-August 2016
-===========
+What would be cool to finish before the end of Milestone 1
+==========================================================
 
-Planned
--------
+
+In-progress ("Lock" section)
+----------------------------
 
 * richard: Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951
   Seems to work, but test suite hangs to verify the CPython tests.
@@ -17,12 +18,9 @@
   module, or the enum module
 
 
-Finished
---------
 
-
-Not in any milestone
---------------------
+Misc stuff not formally in any milestone
+----------------------------------------
 
 * At some point, review lib-python/conftest.py to remove the skips
   due to deadlocks (search for "XXX:")
@@ -44,6 +42,8 @@
   accepting any object whatsoever(?!), which is supposedly a feature
   (see http://bugs.python.org/issue14705).
 
+* ``math.isclose()``
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------

From pypy.commits at gmail.com  Wed Aug 31 05:26:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 02:26:16 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Not a crasher,
 but a doubtful-in-my-opinion design decision
Message-ID: <57c6a2b8.d4e41c0a.41140.c6fd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5697:cb1e43867d2c
Date: 2016-08-31 11:26 +0200
http://bitbucket.org/pypy/extradoc/changeset/cb1e43867d2c/

Log:	Not a crasher, but a doubtful-in-my-opinion design decision

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -40,3 +40,9 @@
 
 * re.sub(b'y', bytearray(b'a'), bytearray(b'xyz')) -> b'xaz'
   re.sub(b'y', bytearray(b'\\n'), bytearray(b'xyz')) -> internal TypeError
+ 
+* not a bug: argument clinic turns the "bool" specifier into
+  PyObject_IsTrue(), accepting any argument whatsoever.  This can easily
+  get very confusing for the user, e.g. after messing up the number of
+  arguments.  For example: os.symlink("/path1", "/path2", "/path3")
+  doesn't fail, it just considers the 3rd argument as some true value.

From pypy.commits at gmail.com  Wed Aug 31 05:27:21 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 31 Aug 2016 02:27:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-byteformat: bytearray can now be formatted
 by %b
Message-ID: <57c6a2f9.05d71c0a.b7e14.c568@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-byteformat
Changeset: r86772:cb92a4ecf35d
Date: 2016-08-31 11:26 +0200
http://bitbucket.org/pypy/pypy/changeset/cb92a4ecf35d/

Log:	bytearray can now be formatted by %b

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -484,7 +484,9 @@
                 self.std_wp(space.bytes_w(w_value))
                 return
             if space.isinstance_w(w_value, space.w_bytearray):
-                self.std_wp(space.bytes_w(w_value))
+                buf = w_value.buffer_w(space, 0)
+                # convert the array of the buffer to a py 2 string
+                self.std_wp(buf.as_str())
                 return
 
             w_bytes_method = space.lookup(w_value, "__bytes__")
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -889,5 +889,6 @@
         assert b'%b' % u'&#12399;&#12356;'.encode('utf-8') == u'&#12399;&#12356;'.encode('utf-8')
         raises(TypeError, 'b"%b" % 3.14')
         raises(TypeError, 'b"%b" % "hello world"')
+        assert b'%b %b' % (b'a', bytearray(b'f f e')) == b'a f f e'
         """
 

From pypy.commits at gmail.com  Wed Aug 31 05:45:07 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 31 Aug 2016 02:45:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-byteformat: added descr_mod to
 W_BytearrayObject + tests, already passing
Message-ID: <57c6a723.04141c0a.2e312.c823@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-byteformat
Changeset: r86773:0f2109d50a32
Date: 2016-08-31 11:44 +0200
http://bitbucket.org/pypy/pypy/changeset/0f2109d50a32/

Log:	added descr_mod to W_BytearrayObject + tests, already passing

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -21,6 +21,7 @@
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.util import get_positive_index
+from pypy.objspace.std.formatting import mod_format, FORMAT_BYTEARRAY
 
 NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d"
 
@@ -446,6 +447,9 @@
     def descr_hex(self, space):
         return _array_to_hexstring(space, self.data, len(self.data), True)
 
+    def descr_mod(self, space, w_values):
+        return mod_format(space, self, w_values, fmt_type=FORMAT_BYTEARRAY)
+
     @staticmethod
     def _iter_getitem_result(self, space, index):
         assert isinstance(self, W_BytearrayObject)
@@ -599,6 +603,9 @@
     def __mul__():
         """x.__mul__(n) <==> x*n"""
 
+    def __mod__():
+        """x.__mod__(y) <==> x % y"""
+
     def __ne__():
         """x.__ne__(y) <==> x!=y"""
 
@@ -1099,6 +1106,8 @@
                              doc=BytearrayDocstrings.__setitem__.__doc__),
     __delitem__ = interp2app(W_BytearrayObject.descr_delitem,
                              doc=BytearrayDocstrings.__delitem__.__doc__),
+    __mod__ = interp2app(W_BytearrayObject.descr_mod,
+                           doc=BytearrayDocstrings.__mod__.__doc__),
 
     append = interp2app(W_BytearrayObject.descr_append,
                         doc=BytearrayDocstrings.append.__doc__),
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -527,7 +527,10 @@
 def format(space, w_fmt, values_w, w_valuedict, fmt_type):
     "Entry point"
     if fmt_type != FORMAT_UNICODE:
-        fmt = space.str_w(w_fmt)
+        if fmt_type == FORMAT_BYTEARRAY:
+            fmt = w_fmt.buffer_w(space, 0).as_str()
+        else:
+            fmt = space.str_w(w_fmt)
         formatter = StringFormatter(space, fmt, values_w, w_valuedict)
         try:
             result = formatter.format()
@@ -538,7 +541,7 @@
             if fmt_type == FORMAT_BYTES:
                 return space.newbytes(result)
             elif fmt_type == FORMAT_BYTEARRAY:
-                return space.newbytearray(result)
+                return space.newbytearray([c for c in result])
             return space.wrap(result)
     fmt = space.unicode_w(w_fmt)
     formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict)
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -1,6 +1,7 @@
+# coding: utf-8
+
 from pypy import conftest
 
-
 class AppTestBytesArray:
     def setup_class(cls):
         cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
@@ -527,3 +528,20 @@
     def test_hex(self):
         assert bytearray(b'santa claus').hex() == "73616e746120636c617573"
 
+    def test_format(self):
+        """
+        assert bytearray(b'a%db') % 2 == b'a2b'
+        assert bytearray(b'00%.2f').__mod__((0.01234,)) == b'000.01'
+        assert bytearray(b'%04X') % 10 == b'000A'
+        assert bytearray(b'%c') % 48 == b'0'
+        assert bytearray(b'%c') % b'a' == b'a'
+        """
+
+    def test_format_b(self):
+        """
+        assert bytearray(b'%b') % b'abc' == b'abc'
+        assert bytearray(b'%b') % u'&#12399;&#12356;'.encode('utf-8') == u'&#12399;&#12356;'.encode('utf-8')
+        raises(TypeError, 'bytearray(b"%b") % 3.14')
+        raises(TypeError, 'bytearray(b"%b") % "hello world"')
+        assert bytearray(b'%b %b') % (b'a', bytearray(b'f f e')) == b'a f f e'
+        """
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -891,4 +891,3 @@
         raises(TypeError, 'b"%b" % "hello world"')
         assert b'%b %b' % (b'a', bytearray(b'f f e')) == b'a f f e'
         """
-

From pypy.commits at gmail.com  Wed Aug 31 06:00:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 03:00:44 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: another to-do
Message-ID: <57c6aacc.915c1c0a.f8309.d79e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5698:741aab547df8
Date: 2016-08-31 12:00 +0200
http://bitbucket.org/pypy/extradoc/changeset/741aab547df8/

Log:	another to-do

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -44,6 +44,10 @@
 
 * ``math.isclose()``
 
+* ``KeyError('pip.exceptions',) in weakref callback <function
+  _get_module_lock.<locals>.cb at 0x00007f118e2c0020> ignored``
+  we're getting them now on start-up, investigate
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------

From pypy.commits at gmail.com  Wed Aug 31 07:37:04 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 31 Aug 2016 04:37:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge byte format for bytes & bytearray
Message-ID: <57c6c160.4825c20a.df1f5.f8ed@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86774:daec70259c2e
Date: 2016-08-31 13:35 +0200
http://bitbucket.org/pypy/pypy/changeset/daec70259c2e/

Log:	merge byte format for bytes & bytearray

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -21,6 +21,7 @@
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.util import get_positive_index
+from pypy.objspace.std.formatting import mod_format, FORMAT_BYTEARRAY
 
 NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d"
 
@@ -446,6 +447,9 @@
     def descr_hex(self, space):
         return _array_to_hexstring(space, self.data, len(self.data), True)
 
+    def descr_mod(self, space, w_values):
+        return mod_format(space, self, w_values, fmt_type=FORMAT_BYTEARRAY)
+
     @staticmethod
     def _iter_getitem_result(self, space, index):
         assert isinstance(self, W_BytearrayObject)
@@ -599,6 +603,9 @@
     def __mul__():
         """x.__mul__(n) <==> x*n"""
 
+    def __mod__():
+        """x.__mod__(y) <==> x % y"""
+
     def __ne__():
         """x.__ne__(y) <==> x!=y"""
 
@@ -1099,6 +1106,8 @@
                              doc=BytearrayDocstrings.__setitem__.__doc__),
     __delitem__ = interp2app(W_BytearrayObject.descr_delitem,
                              doc=BytearrayDocstrings.__delitem__.__doc__),
+    __mod__ = interp2app(W_BytearrayObject.descr_mod,
+                           doc=BytearrayDocstrings.__mod__.__doc__),
 
     append = interp2app(W_BytearrayObject.descr_append,
                         doc=BytearrayDocstrings.append.__doc__),
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -14,6 +14,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
+from pypy.objspace.std.formatting import mod_format, FORMAT_BYTES
 
 
 class W_AbstractBytesObject(W_Root):
@@ -394,6 +395,12 @@
         of the specified width. The string S is never truncated.
         """
 
+    def descr_mod(self, space, w_values):
+        """S % values -> string
+
+        Format bytes objects
+        """
+
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
@@ -663,6 +670,9 @@
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         return _array_to_hexstring(space, StringBuffer(self._value))
 
+    def descr_mod(self, space, w_values):
+        return mod_format(space, self, w_values, fmt_type=FORMAT_BYTES)
+
     @staticmethod
     def _iter_getitem_result(self, space, index):
         assert isinstance(self, W_BytesObject)
@@ -803,6 +813,8 @@
     __mul__ = interpindirect2app(W_AbstractBytesObject.descr_mul),
     __rmul__ = interpindirect2app(W_AbstractBytesObject.descr_rmul),
 
+    __mod__ = interpindirect2app(W_AbstractBytesObject.descr_mod),
+
     __getitem__ = interpindirect2app(W_AbstractBytesObject.descr_getitem),
 
     capitalize = interpindirect2app(W_AbstractBytesObject.descr_capitalize),
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -438,7 +438,7 @@
             self.std_wp(s)
 
         def fmt_r(self, w_value):
-            self.std_wp(self.space.unicode_w(self.space.repr(w_value)))
+            self.fmt_a(w_value)
 
         def fmt_a(self, w_value):
             from pypy.objspace.std.unicodeobject import ascii_from_object
@@ -477,6 +477,31 @@
                                     "character code not in range(256)")
                     self.std_wp(s)
 
+        def fmt_b(self, w_value):
+            space = self.space
+            # cpython explicitly checks for bytes & bytearray
+            if space.isinstance_w(w_value, space.w_bytes):
+                self.std_wp(space.bytes_w(w_value))
+                return
+            if space.isinstance_w(w_value, space.w_bytearray):
+                buf = w_value.buffer_w(space, 0)
+                # convert the array of the buffer to a py 2 string
+                self.std_wp(buf.as_str())
+                return
+
+            w_bytes_method = space.lookup(w_value, "__bytes__")
+            if w_bytes_method is not None:
+                w_bytes = space.get_and_call_function(w_bytes_method, w_value)
+                if not space.isinstance_w(w_bytes, space.w_bytes):
+                    raise oefmt(space.w_TypeError,
+                                "__bytes__ returned non-bytes (type '%T')", w_bytes)
+                self.std_wp(space.bytes_w(w_bytes))
+                return
+
+            raise oefmt(space.w_TypeError,
+                    "requires bytes, or an object that" \
+                    "implements __bytes__, not '%T'", w_value)
+
     return StringFormatter
 
 
@@ -494,10 +519,18 @@
     [_name[-1] for _name in dir(StringFormatter)
                if len(_name) == 5 and _name.startswith('fmt_')])
 
-def format(space, w_fmt, values_w, w_valuedict, do_unicode):
+FORMAT_STR = 0
+FORMAT_UNICODE = 1
+FORMAT_BYTES = 2
+FORMAT_BYTEARRAY = 3
+
+def format(space, w_fmt, values_w, w_valuedict, fmt_type):
     "Entry point"
-    if not do_unicode:
-        fmt = space.str_w(w_fmt)
+    if fmt_type != FORMAT_UNICODE:
+        if fmt_type == FORMAT_BYTEARRAY:
+            fmt = w_fmt.buffer_w(space, 0).as_str()
+        else:
+            fmt = space.str_w(w_fmt)
         formatter = StringFormatter(space, fmt, values_w, w_valuedict)
         try:
             result = formatter.format()
@@ -505,25 +538,29 @@
             # fall through to the unicode case
             pass
         else:
+            if fmt_type == FORMAT_BYTES:
+                return space.newbytes(result)
+            elif fmt_type == FORMAT_BYTEARRAY:
+                return space.newbytearray([c for c in result])
             return space.wrap(result)
     fmt = space.unicode_w(w_fmt)
     formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict)
     result = formatter.format()
     return space.wrap(result)
 
-def mod_format(space, w_format, w_values, do_unicode=False):
+def mod_format(space, w_format, w_values, fmt_type=FORMAT_STR):
     if space.isinstance_w(w_values, space.w_tuple):
         values_w = space.fixedview(w_values)
-        return format(space, w_format, values_w, None, do_unicode)
+        return format(space, w_format, values_w, None, fmt_type)
     else:
         # we check directly for dict to avoid obscure checking
         # in simplest case
         if space.isinstance_w(w_values, space.w_dict) or \
            (space.lookup(w_values, '__getitem__') and
            not space.isinstance_w(w_values, space.w_unicode)):
-            return format(space, w_format, [w_values], w_values, do_unicode)
+            return format(space, w_format, [w_values], w_values, fmt_type)
         else:
-            return format(space, w_format, [w_values], None, do_unicode)
+            return format(space, w_format, [w_values], None, fmt_type)
 
 # ____________________________________________________________
 # Formatting helpers
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -346,6 +346,9 @@
     def newbytes(self, s):
         return W_BytesObject(s)
 
+    def newbytearray(self, l):
+        return W_BytearrayObject(l)
+
     def newunicode(self, uni):
         return W_UnicodeObject(uni)
 
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -1,6 +1,7 @@
+# coding: utf-8
+
 from pypy import conftest
 
-
 class AppTestBytesArray:
     def setup_class(cls):
         cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
@@ -527,3 +528,20 @@
     def test_hex(self):
         assert bytearray(b'santa claus').hex() == "73616e746120636c617573"
 
+    def test_format(self):
+        """
+        assert bytearray(b'a%db') % 2 == b'a2b'
+        assert bytearray(b'00%.2f').__mod__((0.01234,)) == b'000.01'
+        assert bytearray(b'%04X') % 10 == b'000A'
+        assert bytearray(b'%c') % 48 == b'0'
+        assert bytearray(b'%c') % b'a' == b'a'
+        """
+
+    def test_format_b(self):
+        """
+        assert bytearray(b'%b') % b'abc' == b'abc'
+        assert bytearray(b'%b') % u'&#12399;&#12356;'.encode('utf-8') == u'&#12399;&#12356;'.encode('utf-8')
+        raises(TypeError, 'bytearray(b"%b") % 3.14')
+        raises(TypeError, 'bytearray(b"%b") % "hello world"')
+        assert bytearray(b'%b %b') % (b'a', bytearray(b'f f e')) == b'a f f e'
+        """
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 class TestW_BytesObject:
 
     def teardown_method(self, method):
@@ -97,10 +98,6 @@
         assert bytes('abc', 'ascii') == b'abc'
         assert bytes(set(b'foo')) in (b'fo', b'of')
 
-    def test_format(self):
-        import operator
-        raises(TypeError, operator.mod, b"%s", (1,))
-
     def test_fromhex(self):
         assert bytes.fromhex("abcd") == b'\xab\xcd'
         assert b''.fromhex("abcd") == b'\xab\xcd'
@@ -877,3 +874,20 @@
                "73616e746120636c617573"
         assert bytes(64).hex() == "00"*64
 
+    def test_format(self):
+        """
+        assert b'a%db' % 2 == b'a2b'
+        assert b'00%.2f'.__mod__((0.01234,)) == b'000.01'
+        assert b'%04X' % 10 == b'000A'
+        assert b'%c' % 48 == b'0'
+        assert b'%c' % b'a' == b'a'
+        """
+
+    def test_format_b(self):
+        """
+        assert b'%b' % b'abc' == b'abc'
+        assert b'%b' % u'&#12399;&#12356;'.encode('utf-8') == u'&#12399;&#12356;'.encode('utf-8')
+        raises(TypeError, 'b"%b" % 3.14')
+        raises(TypeError, 'b"%b" % "hello world"')
+        assert b'%b %b' % (b'a', bytearray(b'f f e')) == b'a f f e'
+        """
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -15,7 +15,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.module.unicodedata import unicodedb
 from pypy.objspace.std import newformat
-from pypy.objspace.std.formatting import mod_format
+from pypy.objspace.std.formatting import mod_format, FORMAT_UNICODE
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
@@ -392,7 +392,7 @@
                                        self)
 
     def descr_mod(self, space, w_values):
-        return mod_format(space, self, w_values, do_unicode=True)
+        return mod_format(space, self, w_values, fmt_type=FORMAT_UNICODE)
 
     def descr_translate(self, space, w_table):
         selfvalue = self._value

From pypy.commits at gmail.com  Wed Aug 31 07:37:06 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 31 Aug 2016 04:37:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <57c6c162.88cb1c0a.55ab1.fa1f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r86775:0e0dad8e1f2a
Date: 2016-08-31 13:36 +0200
http://bitbucket.org/pypy/pypy/changeset/0e0dad8e1f2a/

Log:	merge heads

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -27,6 +27,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -114,7 +122,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -850,6 +850,9 @@
         raises(TypeError, delattr, A(), 42)
 
     def test_getattr_None(self):
+        import sys
+        if '__pypy__' not in sys.modules:
+            skip('CPython uses wrapper types for this')
         from types import FunctionType, MethodType
         assert isinstance(getattr(type(None), '__eq__'), FunctionType)
         assert isinstance(getattr(None, '__eq__'), MethodType)
@@ -864,4 +867,3 @@
         assert isinstance(getattr(a, '__eq__'), MethodType)
         a.__eq__ = 42
         assert a.__eq__ == 42
-
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -303,6 +303,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -8,6 +8,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rfloat import copysign, formatd
 from rpython.rlib.rarithmetic import r_uint, intmask
+from pypy.interpreter.signature import Signature
 
 
 @specialize.argtype(1)
@@ -40,15 +41,18 @@
 ANS_MANUAL = 3
 
 
-def make_template_formatting_class():
+format_signature = Signature([], 'args', 'kwargs')
+
+
+def make_template_formatting_class(for_unicode):
     class TemplateFormatter(object):
+        is_unicode = for_unicode
 
         parser_list_w = None
 
-        def __init__(self, space, is_unicode, template):
+        def __init__(self, space, template):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.template = template
 
         def build(self, args, w_kwargs):
@@ -198,14 +202,10 @@
             if index == -1:
                 kwarg = name[:i]
                 if self.is_unicode:
-                    try:
-                        arg_key = kwarg.encode("latin-1")
-                    except UnicodeEncodeError:
-                        # Not going to be found in a dict of strings.
-                        raise OperationError(space.w_KeyError, space.wrap(kwarg))
+                    w_kwarg = space.newunicode(kwarg)
                 else:
-                    arg_key = kwarg
-                w_arg = space.getitem(self.w_kwargs, space.wrap(arg_key))
+                    w_kwarg = space.newbytes(kwarg)
+                w_arg = space.getitem(self.w_kwargs, w_kwarg)
             else:
                 if self.args is None:
                     w_msg = space.wrap("Format string contains positional "
@@ -356,14 +356,8 @@
             return space.iter(space.newlist(self.parser_list_w))
     return TemplateFormatter
 
-StrTemplateFormatter = make_template_formatting_class()
-UnicodeTemplateFormatter = make_template_formatting_class()
-
-def str_template_formatter(space, template):
-    return StrTemplateFormatter(space, False, template)
-
-def unicode_template_formatter(space, template):
-    return UnicodeTemplateFormatter(space, True, template)
+str_template_formatter = make_template_formatting_class(for_unicode=False)
+unicode_template_formatter = make_template_formatting_class(for_unicode=True)
 
 
 def format_method(space, w_string, args, w_kwargs, is_unicode):
@@ -400,16 +394,16 @@
 
 LONG_DIGITS = string.digits + string.ascii_lowercase
 
-def make_formatting_class():
+def make_formatting_class(for_unicode):
     class Formatter(BaseFormatter):
         """__format__ implementation for builtin types."""
 
+        is_unicode = for_unicode
         _grouped_digits = None
 
-        def __init__(self, space, is_unicode, spec):
+        def __init__(self, space, spec):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.spec = spec
 
         def _is_alignment(self, c):
@@ -1146,12 +1140,9 @@
             self._unknown_presentation("complex")
     return Formatter
 
-StrFormatter = make_formatting_class()
+unicode_formatter = make_formatting_class(for_unicode=True)
 
 
-def unicode_formatter(space, spec):
-    return StrFormatter(space, True, spec)
-
 @specialize.arg(2)
 def run_formatter(space, w_format_spec, meth, *args):
     formatter = unicode_formatter(space, space.unicode_w(w_format_spec))
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -157,6 +157,7 @@
                 # poor man's x.decode('ascii', 'replace'), since it's not
                 # supported by RPython
                 if not we_are_translated():
+                    import pdb;pdb.set_trace()
                     print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
                 lst = []
                 for ch in x:
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -216,7 +216,9 @@
         assert self.s("{!r}").format(x()) == "32"
 
     def test_non_latin1_key(self):
-        raises(KeyError, self.s("{\u1000}").format)
+        raises(KeyError, u"{\u1000}".format)
+        d = {u"\u1000": u"foo"}
+        assert u"{\u1000}".format(**d) == u"foo"
 
 
 class AppTestBoolFormat:
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -375,12 +375,19 @@
             raise
         return space.newbool(res)
 
+    def _parse_format_arg(self, space, w_kwds, __args__):
+        for i in range(len(__args__.keywords)):
+            try:     # pff
+                arg = __args__.keywords[i].decode('utf-8')
+            except UnicodeDecodeError:
+                continue   # uh, just skip that
+            space.setitem(w_kwds, space.newunicode(arg),
+                          __args__.keywords_w[i])
+
     def descr_format(self, space, __args__):
         w_kwds = space.newdict()
         if __args__.keywords:
-            for i in range(len(__args__.keywords)):
-                space.setitem(w_kwds, space.wrap(__args__.keywords[i]),
-                              __args__.keywords_w[i])
+            self._parse_format_arg(space, w_kwds, __args__)
         return newformat.format_method(space, self, __args__.arguments_w,
                                        w_kwds, True)
 
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -526,7 +526,7 @@
                  fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socket() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.
@@ -655,7 +655,7 @@
         address, addr_p, addrlen_p = self._addrbuf()
         try:
             remove_inheritable = not inheritable
-            if (not inheritable and SOCK_CLOEXEC is not None
+            if (not inheritable and 'SOCK_CLOEXEC' in constants
                     and _c.HAVE_ACCEPT4
                     and _accept4_syscall.attempt_syscall()):
                 newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
@@ -1138,7 +1138,7 @@
         try:
             res = -1
             remove_inheritable = not inheritable
-            if not inheritable and SOCK_CLOEXEC is not None:
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
                 # Non-inheritable: we try to call socketpair() with
                 # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
                 # then we fall back to the SOCK_CLOEXEC-less case.

From pypy.commits at gmail.com  Wed Aug 31 08:13:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 05:13:08 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: update to release 5.4.0
Message-ID: <57c6c9d4.05d71c0a.b7e14.0df9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86776:652ce0f5b981
Date: 2016-08-31 14:12 +0200
http://bitbucket.org/pypy/pypy/changeset/652ce0f5b981/

Log:	update to release 5.4.0

diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -137,6 +137,8 @@
             lib.gdbm_sync(self.__ll_dbm)
 
 def open(filename, flags='r', mode=0666):
+    if isinstance(filename, unicode):
+        filename = filename.encode()
     if flags[0] == 'r':
         iflags = lib.GDBM_READER
     elif flags[0] == 'w':
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -5,8 +5,14 @@
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
 This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
-and fixed many issues and bugs raised by the growing community of PyPy
-users.
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. 
 
 XXXXX MORE ???
 
@@ -25,6 +31,7 @@
 
 .. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
 .. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
 .. _`PyPy`: http://doc.pypy.org
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
@@ -74,13 +81,47 @@
     `issubclass()` as well as `type.__instancecheck__()` and
     `type.__subclasscheck__()`
 
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Support for memoryview attributes (format, itemsize, ...) which also
+    adds support for `PyMemoryView_FromObject`
+
 * Bug Fixes
 
   * Reject `mkdir()` in read-only sandbox filesystems
 
   * Add include guards to pymem.h to enable c++ compilation
 
-  * Fix OpenBSD build breakage and support OpenBSD in VMProf.
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
 
   * Fix for `bytearray('').replace('a', 'ab')` for empty strings
 
@@ -104,10 +145,30 @@
     `MADV_DONTNEED` on freed arenas to release memory back to the
     OS for resource monitoring
 
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
   * Issues reported with our previous release were resolved_ after
     reports from users on our issue tracker at
     https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
 
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
@@ -135,7 +196,23 @@
     RPython functions, eventually exhausting the stack, while at
     app-level the traceback is very short
 
-.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
 
 Please update, and continue to help us make PyPy better.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,161 +1,8 @@
 ==========================
-What's new in PyPy2.7 5.3+
+What's new in PyPy2.7 5.4+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
 
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
-
-
-.. branch: const-fold-we-are-jitted
-
-Reduce the size of the generated C code by constant-folding ``we_are_jitted``
-in non-jitcode.
-
-.. branch: memoryview-attributes
-
-Support for memoryview attributes (format, itemsize, ...).
-Extends the cpyext emulation layer.
-
-.. branch: redirect-assembler-jitlog
-
-Log more information to properly rebuild the redirected traces in jitviewer.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 5.3+
-==========================
+=========================
+What's new in PyPy2.7 5.4
+=========================
 
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
@@ -159,3 +159,7 @@
 .. branch: redirect-assembler-jitlog
 
 Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -48,10 +48,10 @@
             return f.f_code
         assert g() is g.func_code
 
-    def test_f_trace_del(self):
+    def test_f_trace_del(self): 
         import sys
-        f = sys._getframe()
-        del f.f_trace
+        f = sys._getframe() 
+        del f.f_trace 
         assert f.f_trace is None
 
     def test_f_lineno(self):
@@ -116,7 +116,7 @@
         def f():
             assert sys._getframe().f_code.co_name == g()
         def g():
-            return sys._getframe().f_back.f_code.co_name
+            return sys._getframe().f_back.f_code.co_name 
         f()
 
     def test_f_back_virtualref(self):
@@ -233,7 +233,7 @@
     def test_trace_exc(self):
         import sys
         l = []
-        def ltrace(a,b,c):
+        def ltrace(a,b,c): 
             if b == 'exception':
                 l.append(c)
             return ltrace
@@ -298,7 +298,7 @@
     def test_trace_return_exc(self):
         import sys
         l = []
-        def trace(a,b,c):
+        def trace(a,b,c): 
             if b in ('exception', 'return'):
                 l.append((b, c))
             return trace
@@ -444,7 +444,7 @@
     def test_dont_trace_on_reraise(self):
         import sys
         l = []
-        def ltrace(a,b,c):
+        def ltrace(a,b,c): 
             if b == 'exception':
                 l.append(c)
             return ltrace
@@ -466,7 +466,7 @@
     def test_dont_trace_on_raise_with_tb(self):
         import sys
         l = []
-        def ltrace(a,b,c):
+        def ltrace(a,b,c): 
             if b == 'exception':
                 l.append(c)
             return ltrace
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -3,7 +3,7 @@
 from pypy.interpreter import gateway
 from rpython.rlib.jit import non_virtual_ref, vref_None
 
-class AppTestSlow:
+class AppTestSlow:    
     spaceconfig = dict(usemodules=['itertools'])
 
     def setup_class(cls):
@@ -64,7 +64,7 @@
     space.setitem(space.builtin.w_dict,
                   space.wrap('read_exc_type'),
                   space.wrap(read_exc_type_gw))
-
+    
 def _detach_helpers(space):
     space.delitem(space.builtin.w_dict,
                   space.wrap('hide_top_frame'))
@@ -92,7 +92,7 @@
         pckl = pickle.dumps(code)
         result = pickle.loads(pckl)
         assert code == result
-
+    
     def test_pickle_global_func(self):
         import new
         mod = new.module('mod')
@@ -109,7 +109,7 @@
             assert func is result
         finally:
             del sys.modules['mod']
-
+    
     def test_pickle_not_imported_module(self):
         import new
         mod = new.module('mod')
@@ -119,13 +119,13 @@
         result = pickle.loads(pckl)
         assert mod.__name__ == result.__name__
         assert mod.__dict__ == result.__dict__
-
+    
     def test_pickle_builtin_func(self):
         import pickle
         pckl = pickle.dumps(map)
         result = pickle.loads(pckl)
         assert map is result
-
+    
     def test_pickle_non_top_reachable_func(self):
         def func():
             return 42
@@ -142,7 +142,7 @@
         assert func.func_dict     == result.func_dict
         assert func.func_doc      == result.func_doc
         assert func.func_globals  == result.func_globals
-
+    
     def test_pickle_cell(self):
         def g():
             x = [42]
@@ -171,7 +171,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved)
+        restore_top_frame(f1, saved) 
         f2     = pickle.loads(pckl)
 
         assert type(f1) is type(f2)
@@ -223,7 +223,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved)
+        restore_top_frame(f1, saved) 
         f2     = pickle.loads(pckl)
 
     def test_frame_setstate_crash(self):
@@ -257,21 +257,21 @@
         pckl   = pickle.dumps(mod)
         result = pickle.loads(pckl)
         assert mod is result
-
+    
     def test_pickle_moduledict(self):
         import pickle
         moddict  = pickle.__dict__
         pckl     = pickle.dumps(moddict)
         result   = pickle.loads(pckl)
         assert moddict is result
-
+    
     def test_pickle_bltins_module(self):
         import pickle
         mod  = __builtins__
         pckl     = pickle.dumps(mod)
         result   = pickle.loads(pckl)
         assert mod is result
-
+    
     def test_pickle_buffer(self):
         skip("Can't pickle buffer objects on top of CPython either.  "
              "Do we really need it?")
@@ -280,14 +280,14 @@
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-
+    
     def test_pickle_complex(self):
         import pickle
         a = complex(1.23,4.567)
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-
+    
     def test_pickle_method(self):
         class myclass(object):
             def f(self):
@@ -308,7 +308,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-
+    
     def test_pickle_staticmethod(self):
         class myclass(object):
             def f():
@@ -319,7 +319,7 @@
         pckl     = pickle.dumps(method)
         result   = pickle.loads(pckl)
         assert method() == result()
-
+    
     def test_pickle_classmethod(self):
         class myclass(object):
             def f(cls):
@@ -337,7 +337,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-
+    
     def test_pickle_sequenceiter(self):
         '''
         In PyPy there is no distinction here between listiterator and
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,7 +120,7 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 """.split()
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.4.0"
+#define PYPY_VERSION_NUM  0x05040000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -1,3 +1,6 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
 #include <Python.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -10,7 +13,7 @@
 /* Structure defines a 1-dimensional strided array */
 typedef struct{
     int* arr;
-    long length;
+    Py_ssize_t length;
 } MyArray;
 
 /* initialize the array with integers 0...length */
@@ -61,13 +64,13 @@
 static int
 PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
 {
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
     // init may have already been called
     if (self->arr.arr != NULL) {
         deallocate_MyArray(&self->arr);
     }
 
-    int length = 0;
-    static char *kwlist[] = {"length", NULL};
     if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
         return -1;
 
@@ -103,16 +106,19 @@
 static int
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
 
-  PyMyArray* self = (PyMyArray*)obj;
   view->obj = (PyObject*)self;
   view->buf = (void*)self->arr.arr;
   view->len = self->arr.length * sizeof(int);
@@ -218,7 +224,6 @@
 #ifdef __GNUC__
 extern __attribute__((visibility("default")))
 #else
-extern __declspec(dllexport)
 #endif
 
 PyMODINIT_FUNC
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -87,4 +87,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -18,7 +18,8 @@
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER)
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -386,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -814,8 +817,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -143,7 +143,7 @@
 def _setup(space):
     dn = setup_directory_structure(space)
     return space.appexec([space.wrap(dn)], """
-        (dn):
+        (dn): 
             import sys
             path = list(sys.path)
             sys.path.insert(0, dn)
@@ -1017,7 +1017,7 @@
 
         cpathname = udir.join('test.pyc')
         assert not cpathname.check()
-
+        
     def test_load_source_module_importerror(self):
         # the .pyc file is created before executing the module
         space = self.space
@@ -1126,11 +1126,11 @@
                     stream.close()
 
 
-def test_PYTHONPATH_takes_precedence(space):
+def test_PYTHONPATH_takes_precedence(space): 
     if sys.platform == "win32":
         py.test.skip("unresolved issues with win32 shell quoting rules")
-    from pypy.interpreter.test.test_zpy import pypypath
-    extrapath = udir.ensure("pythonpath", dir=1)
+    from pypy.interpreter.test.test_zpy import pypypath 
+    extrapath = udir.ensure("pythonpath", dir=1) 
     extrapath.join("sched.py").write("print 42\n")
     old = os.environ.get('PYTHONPATH', None)
     oldlang = os.environ.pop('LANG', None)
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -495,8 +495,8 @@
         c = array([1.e+110, 1.e-110], dtype=complex128)
         d = floor_divide(c**2, c)
         assert (d == [1.e+110, 0]).all()
-
-
+        
+        
 
     def test_basic(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -372,8 +372,8 @@
         a = np.array(data, dtype=b)
         x = pickle.loads(pickle.dumps(a))
         assert (x == a).all()
-        assert x.dtype == a.dtype
-
+        assert x.dtype == a.dtype 
+        
     def test_index(self):
         import numpy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:
@@ -1459,7 +1459,7 @@
                      "'offsets':[0,76800], "
                      "'itemsize':80000, "
                      "'aligned':True}")
-
+        
         assert dt == np.dtype(eval(str(dt)))
 
         dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'],
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1878,7 +1878,7 @@
         assert map(isnan, e) == [False, False, False, True, False]
         assert map(isinf, e) == [False, False, True, False, False]
         assert e.argmax() == 3
-        # numpy preserves value for uint16 -> cast_as_float16 ->
+        # numpy preserves value for uint16 -> cast_as_float16 -> 
         #     convert_to_float64 -> convert_to_float16 -> uint16
         #  even for float16 various float16 nans
         all_f16 = arange(0xfe00, 0xffff, dtype='uint16')
@@ -2608,7 +2608,7 @@
         a = np.arange(6).reshape(2,3)
         i = np.dtype('int32').type(0)
         assert (a[0] == a[i]).all()
-
+        
 
     def test_ellipsis_indexing(self):
         import numpy as np
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -200,7 +200,7 @@
         from numpy import arange, dtype
         from cPickle import loads, dumps
         import sys
-
+        
         a = arange(15).astype(object)
         if '__pypy__' in sys.builtin_module_names:
             raises(NotImplementedError, dumps, a)
@@ -211,4 +211,4 @@
         a = arange(15).astype(object).reshape((3, 5))
         b = loads(dumps(a))
         assert (a == b).all()
-
+        
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 2, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 4, 0, "final", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/module/test_lib_pypy/test_gdbm_extra.py b/pypy/module/test_lib_pypy/test_gdbm_extra.py
--- a/pypy/module/test_lib_pypy/test_gdbm_extra.py
+++ b/pypy/module/test_lib_pypy/test_gdbm_extra.py
@@ -15,3 +15,7 @@
     assert len(g) == 2
     del g['abc']
     assert len(g) == 1
+
+def test_unicode():
+    path = unicode(udir.join('test_gdm_unicode'))
+    g = gdbm.open(path, 'c')  # does not crash
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -1,7 +1,10 @@
 import pytest
 import sys
-from hypothesis import given, strategies, settings
 from pypy.tool.pytest.objspace import gettestobjspace
+try:
+    from hypothesis import given, strategies, settings
+except ImportError:
+    pytest.skip("requires hypothesis")
 
 base_initargs = strategies.sampled_from([
     ("object", (), False),
diff --git a/pypy/objspace/test/test_binop_overriding.py b/pypy/objspace/test/test_binop_overriding.py
--- a/pypy/objspace/test/test_binop_overriding.py
+++ b/pypy/objspace/test/test_binop_overriding.py
@@ -73,7 +73,7 @@
                 if C is not object:
                     setattr(C, name, f)
                 override_in_hier(n-1)
-                if C is not object:
+                if C is not object:        
                     delattr(C, name)
 
         override_in_hier()
@@ -105,7 +105,7 @@
         if not self.appdirect:
             skip("slow test, should be run as appdirect test")
         Base, do_test = self.helpers
-
+         
         class X(Base):
             pass
         class Y(X):
@@ -116,7 +116,7 @@
         assert not fail
 
     def test_binop_combinations_sub(self):
-        Base, do_test = self.helpers
+        Base, do_test = self.helpers        
         class X(Base):
             pass
         class Y(X):
@@ -124,13 +124,13 @@
 
         fail = do_test(X, Y, 'sub', lambda x,y: x-y)
         #print len(fail)
-        assert not fail
+        assert not fail        
 
     def test_binop_combinations_pow(self):
         if not self.appdirect:
             skip("slow test, should be run as appdirect test")
         Base, do_test = self.helpers
-
+        
         class X(Base):
             pass
         class Y(X):
@@ -138,13 +138,13 @@
 
         fail = do_test(X, Y, 'pow', lambda x,y: x**y)
         #print len(fail)
-        assert not fail
+        assert not fail        
 
     def test_binop_combinations_more_exhaustive(self):
         if not self.appdirect:
             skip("very slow test, should be run as appdirect test")
         Base, do_test = self.helpers
-
+        
         class X(Base):
             pass
 
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=3
-rev=1
+min=4
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -11,7 +11,7 @@
 
 class FakeCallAssemblerLoopToken(AbstractDescr):
     def __init__(self, target):
-	self._ll_function_addr = target
+        self._ll_function_addr = target
 
     def repr_of_descr(self):
         return 'looptoken'

From pypy.commits at gmail.com  Wed Aug 31 09:34:49 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 31 Aug 2016 06:34:49 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Try forcing virtual objects
 to avoid jumping to preamble
Message-ID: <57c6dcf9.861b1c0a.2c8d3.3e7a@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86777:cd41860150a5
Date: 2016-08-30 15:08 -0400
http://bitbucket.org/pypy/pypy/changeset/cd41860150a5/

Log:	Try forcing virtual objects to avoid jumping to preamble

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,7 +16,7 @@
 from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
      have_debug_prints
 
-class UnrollableOptimizer(Optimizer):    
+class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
         if isinstance(preamble_op, PreambleOp):
             if self.optunroll.short_preamble_producer is None:
@@ -120,7 +120,8 @@
                 assert op.get_forwarded() is None
         if check_newops:
             assert not self.optimizer._newoperations
-    
+
+
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
@@ -156,7 +157,7 @@
         current_vs = self.get_virtual_state(end_jump.getarglist())
         # pick the vs we want to jump to
         assert isinstance(celltoken, JitCellToken)
-        
+
         target_virtual_state = self.pick_virtual_state(current_vs,
                                                        state.virtual_state,
                                                 celltoken.target_tokens)
@@ -180,7 +181,7 @@
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
-                    self.optimizer._newoperations)            
+                    self.optimizer._newoperations)
 
         try:
             new_virtual_state = self.jump_to_existing_trace(end_jump, label_op,
@@ -191,6 +192,16 @@
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
                     self.optimizer._newoperations)
+
+        if new_virtual_state is not None:
+            # Attempt to force virtual boxes in order to avoid jumping
+            # to the preamble.
+            try:
+                new_virtual_state = self.jump_to_existing_trace(
+                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+            except InvalidLoop:
+                pass
+
         if new_virtual_state is not None:
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
@@ -199,7 +210,7 @@
 
         self.disable_retracing_if_max_retrace_guards(
             self.optimizer._newoperations, target_token)
-        
+
         return (UnrollInfo(target_token, label_op, extra_same_as,
                            self.optimizer.quasi_immutable_deps),
                 self.optimizer._newoperations)
@@ -241,7 +252,7 @@
         for a in jump_op.getarglist():
             self.optimizer.force_box_for_end_of_preamble(a)
         try:
-            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes)
+            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes, False)
         except InvalidLoop:
             return self.jump_to_preamble(cell_token, jump_op, info)
         if vs is None:
@@ -252,6 +263,13 @@
             cell_token.retraced_count += 1
             debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
         else:
+            # Try forcing boxes to avoid jumping to the preamble
+            try:
+                vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes, True)
+            except InvalidLoop:
+                pass
+            if vs is None:
+                return info, self.optimizer._newoperations[:]
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
@@ -288,7 +306,7 @@
         return info, self.optimizer._newoperations[:]
 
 
-    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes):
+    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False):
         jitcelltoken = jump_op.getdescr()
         assert isinstance(jitcelltoken, JitCellToken)
         virtual_state = self.get_virtual_state(jump_op.getarglist())
@@ -299,17 +317,18 @@
                 continue
             try:
                 extra_guards = target_virtual_state.generate_guards(
-                    virtual_state, args, runtime_boxes, self.optimizer)
+                    virtual_state, args, runtime_boxes, self.optimizer,
+                    force_boxes=force_boxes)
                 patchguardop = self.optimizer.patchguardop
                 for guard in extra_guards.extra_guards:
                     if isinstance(guard, GuardResOp):
                         guard.rd_resume_position = patchguardop.rd_resume_position
                         guard.setdescr(compile.ResumeAtPositionDescr())
                     self.send_extra_operation(guard)
-            except VirtualStatesCantMatch:
+            except VirtualStatesCantMatch as e:
                 continue
             args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
-                args, self.optimizer)
+                args, self.optimizer, force_boxes=force_boxes)
             short_preamble = target_token.short_preamble
             try:
                 extra = self.inline_short_preamble(args + virtuals, args,
@@ -452,7 +471,7 @@
         # by short preamble
         label_args = exported_state.virtual_state.make_inputargs(
             targetargs, self.optimizer)
-        
+
         self.short_preamble_producer = ShortPreambleBuilder(
             label_args, exported_state.short_boxes,
             exported_state.short_inputargs, exported_state.exported_infos,
@@ -497,7 +516,7 @@
     * runtime_boxes - runtime values for boxes, necessary when generating
                       guards to jump to
     """
-    
+
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
                  short_inputargs, runtime_boxes, memo):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -4,7 +4,7 @@
      ArrayStructInfo, AbstractStructPtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound, IntLowerBound
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+from rpython.jit.metainterp.resoperation import rop, ResOperation, \
      InputArgInt, InputArgRef, InputArgFloat
 from rpython.rlib.debug import debug_print
 
@@ -20,7 +20,7 @@
 
 
 class GenerateGuardState(object):
-    def __init__(self, optimizer=None, guards=None, renum=None, bad=None):
+    def __init__(self, optimizer=None, guards=None, renum=None, bad=None, force_boxes=False):
         self.optimizer = optimizer
         self.cpu = optimizer.cpu
         if guards is None:
@@ -32,6 +32,7 @@
         if bad is None:
             bad = {}
         self.bad = bad
+        self.force_boxes = force_boxes
 
     def get_runtime_item(self, box, descr, i):
         array = box.getref_base()
@@ -303,7 +304,7 @@
             opinfo = state.optimizer.getptrinfo(box)
             assert isinstance(opinfo, ArrayPtrInfo)
         else:
-            opinfo = None            
+            opinfo = None
         for i in range(self.length):
             for descr in self.fielddescrs:
                 index = i * len(self.fielddescrs) + descr.get_index()
@@ -514,6 +515,8 @@
         NotVirtualStateInfo.__init__(self, cpu, type, info)
 
     def _generate_guards(self, other, box, runtime_box, state):
+        if state.force_boxes and isinstance(other, VirtualStateInfo):
+            return self._generate_virtual_guards(other, box, runtime_box, state)
         if not isinstance(other, NotVirtualStateInfoPtr):
             raise VirtualStatesCantMatch(
                     'The VirtualStates does not match as a ' +
@@ -545,6 +548,23 @@
     # to an existing compiled loop or retracing the loop. Both alternatives
     # will always generate correct behaviour, but performance will differ.
 
+    def _generate_virtual_guards(self, other, box, runtime_box, state):
+        """
+        Generate the guards and add state information for unifying a virtual
+        object with a non-virtual. This involves forcing the object in the
+        event that unifcation can succeed. Since virtual objects cannot be null,
+        this method need only check that the virtual object has the expected type.
+        """
+        assert isinstance(other, VirtualStateInfo)
+
+        if self.level == LEVEL_CONSTANT:
+            raise VirtualStatesCantMatch(
+                    "cannot unify a constant value with a virtual object")
+
+        if self.level == LEVEL_KNOWNCLASS:
+            if not self.known_class.same_constant(other.known_class):
+                raise VirtualStatesCantMatch("classes don't match")
+
     def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards,
                                  state):
         if not isinstance(other, NotVirtualStateInfoPtr):
@@ -617,10 +637,10 @@
             return False
         return True
 
-    def generate_guards(self, other, boxes, runtime_boxes, optimizer):
+    def generate_guards(self, other, boxes, runtime_boxes, optimizer, force_boxes=False):
         assert (len(self.state) == len(other.state) == len(boxes) ==
                 len(runtime_boxes))
-        state = GenerateGuardState(optimizer)
+        state = GenerateGuardState(optimizer, force_boxes=force_boxes)
         for i in range(len(self.state)):
             self.state[i].generate_guards(other.state[i], boxes[i],
                                           runtime_boxes[i], state)
@@ -644,8 +664,8 @@
 
         return boxes
 
-    def make_inputargs_and_virtuals(self, inputargs, optimizer):
-        inpargs = self.make_inputargs(inputargs, optimizer)
+    def make_inputargs_and_virtuals(self, inputargs, optimizer, force_boxes=False):
+        inpargs = self.make_inputargs(inputargs, optimizer, force_boxes)
         # we append the virtuals here in case some stuff is proven
         # to be not a virtual and there are getfields in the short preamble
         # that will read items out of there
@@ -653,7 +673,7 @@
         for i in range(len(inputargs)):
             if not isinstance(self.state[i], NotVirtualStateInfo):
                 virtuals.append(inputargs[i])
-            
+
         return inpargs, virtuals
 
     def debug_print(self, hdr='', bad=None, metainterp_sd=None):
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4507,3 +4507,33 @@
                 i += 1
             return i
         self.meta_interp(f, [])
+
+    def test_loop_unroll_bug(self):
+        driver = JitDriver(greens=[], reds=['acc', 'i', 'val'])
+        class X(object):
+            # _immutable_ = True
+            def __init__(self, v):
+                self.v = v
+
+        class Box(object):
+            def __init__(self, v):
+                self.unbox = v
+
+        const = Box(X(5))
+        def f(v):
+            val   = X(0)
+            acc   = 0
+            i     = 0
+            const.unbox = X(5)
+            while i < 100:
+                driver.can_enter_jit(acc=acc, i=i, val=val)
+                driver.jit_merge_point(acc=acc, i=i, val=val)
+                acc += val.v
+                if i & 0b100 == 0:
+                    val = const.unbox
+                else:
+                    val = X(i)
+                i += 1
+            return acc
+        result = self.meta_interp(f, [10])
+        # import pdb; pdb.set_trace()

From pypy.commits at gmail.com  Wed Aug 31 10:35:01 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 Aug 2016 07:35:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix translation
Message-ID: <57c6eb15.151a1c0a.d0458.4658@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86778:eb9659774e72
Date: 2016-08-31 15:34 +0100
http://bitbucket.org/pypy/pypy/changeset/eb9659774e72/

Log:	fix translation

diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -29,10 +29,10 @@
 
     def descr_repr(self, space):
         if self.w_objtype is not None:
-            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+            objtype_name = u"<%s object>" % self.w_objtype.getname(space)
         else:
-            objtype_name = 'NULL'
-        return space.wrap("<super: <class '%s'>, %s>" % (
+            objtype_name = u'NULL'
+        return space.wrap(u"<super: <class '%s'>, %s>" % (
             self.w_starttype.getname(space), objtype_name))
 
     def get(self, space, w_obj, w_type=None):

From pypy.commits at gmail.com  Wed Aug 31 10:48:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 07:48:36 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Copy this from pypy-z
Message-ID: <57c6ee44.12331c0a.ee8e2.4960@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5699:19ee8f38e22f
Date: 2016-08-31 16:48 +0200
http://bitbucket.org/pypy/extradoc/changeset/19ee8f38e22f/

Log:	Copy this from pypy-z

diff --git a/planning/py3.5/py35plan-milestones.rst b/planning/py3.5/py35plan-milestones.rst
new file mode 100644
--- /dev/null
+++ b/planning/py3.5/py35plan-milestones.rst
@@ -0,0 +1,133 @@
+
+
+Milestone 1
+-----------
+
+async/await support, and some other general Python 3.4/3.5 features.
+
+For the whole duration of the project we will certainly work on the very
+large set of small-scale new features.  These features are listed in
+a lot of details in these pages:
+
+* https://docs.python.org/3/whatsnew/3.4.html
+
+* https://docs.python.org/3/whatsnew/3.5.html
+
+Milestone 1 contains only a subset of these features.  We have reached
+milestone 1 when we have done all the following point, possibly minus
+one of them if it is found during development that properly
+implementing it requires significantly more efforts than planned:
+
+* PEP 492, coroutines with async and await syntax.  (The complete PEP
+  is included.)
+
+* PEP 465, a new matrix multiplication operator: a @ b.
+
+* PEP 448, additional unpacking generalizations.
+
+* bytes % args, bytearray % args: PEP 461
+
+* New bytes.hex(), bytearray.hex() and memoryview.hex() methods.
+
+* memoryview now supports tuple indexing
+
+* Generators have a new gi_yieldfrom attribute
+
+* A new RecursionError exception is now raised when maximum recursion
+  depth is reached.
+
+* The new os.scandir() function
+
+* Newly created file descriptors are non-inheritable (PEP 446)
+
+* The marshal format has been made more compact and efficient
+
+* enum: Support for enumeration types (PEP 435).
+
+* pathlib: Object-oriented filesystem paths (PEP 428).
+
+This includes checking and integrating the code produced by the Summer
+of Code student that starts working on some of these features, namely
+the first three points above.
+
+
+
+Milestone 2
+-----------
+
+Changes to the C API.
+
+We get an up-to-date ``cpyext`` module that supports CPython 3.5 C
+extension modules, including "argument clinic" and other parts of
+the C API that are new.  The goal is that ``cpyext`` works as well
+as it does on PyPy2.
+
+Additionaly we resolve several security related issues found in CPython 3.4/3.5:
+
+* Secure and interchangeable hash algorithm (PEP 456).
+
+* New command line option for isolated mode.
+
+* Enhancements to multiprocessing modules.
+
+* HTTP cookie parsing is now stricter (issue 22796).
+
+The measure of when this milestone is reached is based on the
+following criteria: we can take a number of C extension modules that
+work on CPython 3.5 (without reaching into the internals, like a few
+modules do), and check that they work on PyPy 3.5 as well.  More
+specifically, for any C module with a 2.7 version that works on PyPy
+2.7, its 3.5 equivalent version must also work on PyPy 3.5.
+
+
+Milestone 3
+-----------
+
+ssl and more sustainable stdlib modules with cffi.  Compact unicode
+representation.
+
+We get an SSL module compatible with Python 3.5 (more specifically the
+latest 3.5.x micro release).  The details of what's new in CPython 3.5
+about the SSL module are documented in the two pages linked above.
+
+We also make sure that CFFI works on PyPy 3.5.  At this point either
+our SSL module is an extension of the SSL module that exists in PyPy2,
+or it was rewritten using CFFI, if we decided that this is the way to
+go.
+
+In this milestone is also present the other topic whose details are
+still open: to make unicode strings more compact, we may use either
+CPython's approach (https://docs.python.org/3/whatsnew/3.3.html#pep-393)
+or UTF-8 strings.
+
+Furthermore we will update and resolve more security related issues.
+In addition to the ssl related changes we solve the following:
+
+* A new hashlib.pbkdf2_hmac() function.
+
+* TLSv1.1 and TLSv1.2 support for ssl.
+
+* Server-side SNI (Server Name Indication) support for ssl.
+
+* Server certificate verification, including hostname matching and CRLs.
+
+* SSLv3 is now disabled throughout the standard library (issue 22638).
+
+Milestone 4
+===========
+
+Benchmarking and performance improvements.
+
+The majority of work in Milestone 4 will be focused on delivering the performance
+improvements, which also requires porting PyPy benchmark suite to Python 3. The vast
+majority of large library-based benchmarks that are in the PyPy benchmark suite run on
+Python 3 these days, so porting of the benchmarks should not be a major problem.
+
+There are a few optimizations that were either disabled in PyPy3 or never ported,
+together with some stuff that requires new thinking w.r.t unicode used a bit more
+internally in the interpreter source. After porting the benchmark
+suite we expect to find them and fix them.
+
+The concrete deliverable will be a release of compliant and performant
+PyPy3.5 version together with a blog post describing what we have
+done.

From pypy.commits at gmail.com  Wed Aug 31 10:54:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 07:54:15 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Breakpoints at given line
 numbers: starting
Message-ID: <57c6ef97.e2efc20a.1a16b.18e9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86779:7166e102d539
Date: 2016-08-31 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/7166e102d539/

Log:	Breakpoints at given line numbers: starting

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -1,5 +1,5 @@
 import sys
-from rpython.rlib import revdb
+from rpython.rlib import revdb, rpath, rstring
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
@@ -14,6 +14,7 @@
     standard_code = True
     breakpoint_stack_id = 0
     breakpoint_funcnames = None
+    breakpoint_filelines = None
     printed_objects = {}
     metavars = []
     watch_progs = []
@@ -512,22 +513,101 @@
 lambda_locals = lambda: command_locals
 
 
+def valid_identifier(s):
+    if not s:
+        return False
+    if s[0].isdigit():
+        return False
+    for c in s:
+        if not (c.isalnum() or c == '_'):
+            return False
+    return True
+
+def add_breakpoint_funcname(name, i):
+    if dbstate.breakpoint_funcnames is None:
+        dbstate.breakpoint_funcnames = {}
+    dbstate.breakpoint_funcnames[name] = i
+
+def add_breakpoint_fileline(filename, lineno, i):
+    if dbstate.breakpoint_filelines is None:
+        dbstate.breakpoint_filelines = {}
+    linenos = dbstate.breakpoint_filelines.setdefault(filename, {})
+    linenos[lineno] = i
+
+def add_breakpoint(name, i):
+    # if it is empty, complain
+    if not name:
+        revdb.send_output("Empty breakpoint name\n")
+        return
+    # if it is surrounded by < >, it is the name of a code object
+    if name.startswith('<') and name.endswith('>'):
+        add_breakpoint_funcname(name, i)
+        return
+    # if it has no ':', it can be a valid identifier (which we
+    # register as a function name), or a lineno
+    original_name = name
+    if ':' not in name:
+        try:
+            lineno = int(name)
+        except ValueError:
+            if not valid_identifier(name):
+                revdb.send_output(
+                    'Note: "%s()" doesn''t look like a function name. '
+                    'Setting breakpoint anyway\n' % (name,))
+            add_breakpoint_funcname(name, i)
+            return
+        # "number" does the same as ":number"
+        filename = ''
+    else:
+        # if it has a ':', it must end in ':lineno'
+        j = name.rfind(':')
+        try:
+            lineno = int(name[j+1:])
+        except ValueError:
+            revdb.send_output('"%s": expected a line number after colon\n' % (
+                name,))
+            return
+        filename = name[:j]
+
+    # the text before must be a pathname, possibly a relative one,
+    # or be escaped by < >.  if it isn't, make it absolute and normalized
+    # and warn if it doesn't end in '.py'.
+    if filename == '':
+        frame = fetch_cur_frame()
+        if frame is None:
+            return
+        filename = frame.getcode().co_filename
+    elif filename.startswith('<') and filename.endswith('>'):
+        pass    # use unmodified
+    elif not filename.lower().endswith('.py'):
+        # use unmodified, but warn
+        revdb.send_output(
+            'Note: "%s" doesn''t look like a co_filename. '
+            'Setting breakpoint anyway\n' % (filename,))
+    elif '\x00' not in filename:
+        filename = rstring.assert_str0(filename)
+        filename = rpath.rabspath(filename)
+        filename = rpath.rnormpath(filename)
+
+    add_breakpoint_fileline(filename, lineno, i)
+    name = '%s:%d' % (filename, lineno)
+    if name != original_name:
+        revdb.send_change_breakpoint(i, name)
+
 def command_breakpoints(cmd, extra):
     space = dbstate.space
     dbstate.breakpoint_stack_id = cmd.c_arg1
     revdb.set_thread_breakpoint(cmd.c_arg2)
-    funcnames = None
+    dbstate.breakpoint_funcnames = None
+    dbstate.breakpoint_filelines = None
     watch_progs = []
     with non_standard_code:
         for i, kind, name in revdb.split_breakpoints_arg(extra):
             if kind == 'B':
-                if funcnames is None:
-                    funcnames = {}
-                funcnames[name] = i
+                add_breakpoint(name, i)
             elif kind == 'W':
                 code = interp_marshal.loads(space, space.wrap(name))
                 watch_progs.append((code, i, ''))
-    dbstate.breakpoint_funcnames = funcnames
     dbstate.watch_progs = watch_progs[:]
 lambda_breakpoints = lambda: command_breakpoints
 
diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
--- a/pypy/interpreter/test/test_reverse_debugging.py
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -1,5 +1,7 @@
 import dis
 from pypy.interpreter.reverse_debugging import *
+from pypy.interpreter import reverse_debugging
+from rpython.rlib import revdb, rpath
 from hypothesis import given, strategies, example
 
 
@@ -34,3 +36,83 @@
         assert lstart[index] == chr(next_start - index
                                     if next_start - index <= 255
                                     else 255)
+
+class FakeFrame:
+    def __init__(self, code):
+        self.__code = code
+    def getcode(self):
+        return self.__code
+
+class FakeCode:
+    def __init__(self, co_filename):
+        self.co_filename = co_filename
+
+def check_add_breakpoint(input, curfilename=None,
+                         expected_funcname=None,
+                         expected_fileline=None,
+                         expected_output=None,
+                         expected_chbkpt=None):
+    dbstate.__dict__.clear()
+    prev = revdb.send_answer, reverse_debugging.fetch_cur_frame
+    try:
+        messages = []
+        def got_message(cmd, arg1=0, arg2=0, arg3=0, extra=""):
+            messages.append((cmd, arg1, arg2, arg3, extra))
+        def my_cur_frame():
+            assert curfilename is not None
+            return FakeFrame(FakeCode(curfilename))
+        revdb.send_answer = got_message
+        reverse_debugging.fetch_cur_frame = my_cur_frame
+        add_breakpoint(input, 5)
+    finally:
+        revdb.send_answer, reverse_debugging.fetch_cur_frame = prev
+
+    if expected_funcname is None:
+        assert dbstate.breakpoint_funcnames is None
+    else:
+        assert dbstate.breakpoint_funcnames == {expected_funcname: 5}
+
+    if expected_fileline is None:
+        assert dbstate.breakpoint_filelines is None
+    else:
+        filename, lineno = expected_fileline
+        assert dbstate.breakpoint_filelines == {filename: {lineno: 5}}
+
+    got_output = None
+    got_chbkpt = None
+    if messages:
+        assert len(messages) <= 1
+        if messages[0][0] == revdb.ANSWER_TEXT:
+            got_output = messages[0][-1]
+        if messages[0][0] == revdb.ANSWER_CHBKPT:
+            assert messages[0][1] == 5
+            got_chbkpt = messages[0][-1]
+
+    assert got_output == expected_output
+    assert got_chbkpt == expected_chbkpt
+
+def fullpath(path):
+    return rpath.rnormpath(rpath.rabspath(path))
+
+def test_add_breakpoint():
+    check_add_breakpoint('', expected_output="Empty breakpoint name\n")
+    check_add_breakpoint('foo42', expected_funcname="foo42")
+    check_add_breakpoint('foo.bar', expected_funcname="foo.bar",
+        expected_output='Note: "foo.bar()" doesn''t look like a function name.'
+                        ' Setting breakpoint anyway\n')
+    check_add_breakpoint('<foo.bar>', expected_funcname="<foo.bar>")
+    check_add_breakpoint('42', curfilename='abcd',
+                         expected_fileline=('abcd', 42),
+                         expected_chbkpt='abcd:42')
+    check_add_breakpoint(':42', curfilename='abcd',
+                         expected_fileline=('abcd', 42),
+                         expected_chbkpt='abcd:42')
+    check_add_breakpoint('abcd:42', expected_fileline=('abcd', 42),
+        expected_output='Note: "abcd" doesnt look like a co_filename.'
+                        ' Setting breakpoint anyway\n')
+    full = fullpath('abcd.py')
+    check_add_breakpoint('abcd.py:42',
+                         expected_fileline=(full, 42),
+                         expected_chbkpt='%s:42' % full)
+    check_add_breakpoint('%s:42' % full,
+                         expected_fileline=(full, 42))
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -24,6 +24,7 @@
 ANSWER_STACKID  = 21
 ANSWER_NEXTNID  = 22
 ANSWER_WATCH    = 23
+ANSWER_CHBKPT   = 24
 
 
 def stop_point(place=0):
@@ -54,6 +55,9 @@
 def send_linecache(filename, linenum, strip=True):
     send_answer(ANSWER_LINECACHE, linenum, int(strip), extra=filename)
 
+def send_change_breakpoint(breakpointnum, newtext):
+    send_answer(ANSWER_CHBKPT, breakpointnum, extra=newtext)
+
 def current_time():
     """For RPython debug commands: returns the current time."""
     return llop.revdb_get_value(lltype.SignedLongLong, 'c')

From pypy.commits at gmail.com  Wed Aug 31 10:54:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 07:54:19 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: in-progress: activate
 breakpoints based on line number
Message-ID: <57c6ef9b.e6ecc20a.c6e0d.18a6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86781:99b8ccb0e43a
Date: 2016-08-31 16:53 +0200
http://bitbucket.org/pypy/pypy/changeset/99b8ccb0e43a/

Log:	in-progress: activate breakpoints based on line number

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -15,6 +15,7 @@
     breakpoint_stack_id = 0
     breakpoint_funcnames = None
     breakpoint_filelines = None
+    breakpoint_version = 0
     printed_objects = {}
     metavars = []
     watch_progs = []
@@ -24,6 +25,8 @@
 
 
 pycode.PyCode.co_revdb_linestarts = None   # or a string: see below
+pycode.PyCode.co_revdb_bkpt_version = 0    # see check_and_trigger_bkpt()
+pycode.PyCode.co_revdb_bkpt_cache = None   # see check_and_trigger_bkpt()
 
 # invariant: "f_revdb_nextline_instr" is the bytecode offset of
 # the start of the line that follows "last_instr".
@@ -128,6 +131,8 @@
             call_stop_point_at_line = False
     #
     if call_stop_point_at_line:
+        if dbstate.breakpoint_filelines is not None:
+            check_and_trigger_bkpt(frame.pycode, cur)
         stop_point_activate()
         cur += 1
         ch = ord(co_revdb_linestarts[cur])
@@ -192,6 +197,25 @@
         p += 2
     return lineno
 
+def find_line_starts(code):
+    # RPython version of dis.findlinestarts()
+    lnotab = code.co_lnotab
+    lastlineno = -1
+    lineno = code.co_firstlineno
+    addr = 0
+    p = 0
+    result = []
+    while p < len(lnotab) - 1:
+        byte_incr = ord(lnotab[p])
+        if byte_incr:
+            if lineno != lastlineno:
+                result.append((addr, lineno))
+                lastlineno = lineno
+            addr += byte_incr
+        lineno += line_incr
+    if lineno != lastlineno:
+        result.append((addr, lineno))
+    return result
 
 class NonStandardCode(object):
     def __enter__(self):
@@ -512,6 +536,47 @@
             revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))
 lambda_locals = lambda: command_locals
 
+# ____________________________________________________________
+
+
+def check_and_trigger_bkpt(pycode, opindex):
+    # We cache on 'pycode.co_revdb_bkpt_cache' either None or a dict
+    # mapping {opindex: bkpt_num}.  This cache is updated when the
+    # version in 'pycode.co_revdb_bkpt_version' does not match
+    # 'dbstate.breakpoint_version' any more.
+    if pycode.co_revdb_bkpt_version != dbstate.breakpoint_version:
+        update_bkpt_cache(pycode)
+    cache = pycode.co_revdb_bkpt_cache
+    if cache is not None and opindex in cache:
+        revdb.breakpoint(cache[opindex])
+
+def update_bkpt_cache(pycode):
+    # dbstate.breakpoint_filelines == {'normfilename': {lineno: bkpt_num}}
+    co_filename = pycode.co_filename
+    try:
+        linenos = dbstate.breakpoint_filelines[co_filename]
+    except KeyError:
+        # normalize co_filename, and assigns the {lineno: bkpt_num} dict
+        # back over the original key, to avoid calling rabspath/rnormpath
+        # again the next time
+        normfilename = rpath.rabspath(co_filename)
+        normfilename = rpath.rnormpath(normfilename)
+        linenos = dbstate.breakpoint_filelines.get(normfilename, None)
+        dbstate.breakpoint_filelines[co_filename] = linenos
+    #
+    newcache = None
+    if linenos is not None:
+        # parse co_lnotab to figure out the opindexes that correspond
+        # to the marked line numbers.
+        for addr, lineno in find_line_starts(pycode):
+            if lineno in linenos:
+                if newcache is None:
+                    newcache = {}
+                newcache[addr] = linenos[lineno]
+    #
+    pycode.co_revdb_bkpt_cache = newcache
+    pycode.co_revdb_bkpt_version = dbstate.breakpoint_version
+
 
 def valid_identifier(s):
     if not s:
@@ -600,6 +665,7 @@
     revdb.set_thread_breakpoint(cmd.c_arg2)
     dbstate.breakpoint_funcnames = None
     dbstate.breakpoint_filelines = None
+    dbstate.breakpoint_version += 1
     watch_progs = []
     with non_standard_code:
         for i, kind, name in revdb.split_breakpoints_arg(extra):

From pypy.commits at gmail.com  Wed Aug 31 10:54:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 07:54:17 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fix
Message-ID: <57c6ef99.a699c20a.31fdb.1dd8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86780:fc28ecc9228c
Date: 2016-08-31 16:53 +0200
http://bitbucket.org/pypy/pypy/changeset/fc28ecc9228c/

Log:	Fix

diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
--- a/pypy/interpreter/test/test_reverse_debugging.py
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -7,11 +7,12 @@
 
 class FakeCode:
     hidden_applevel = False
-    def __init__(self, co_code, co_lnotab):
+    def __init__(self, co_code='', co_lnotab='', co_filename='?'):
         self.co_firstlineno = 43
         self.co_code = co_code
         self.co_lnotab = co_lnotab
         self.co_revdb_linestarts = None
+        self.co_filename = co_filename
 
 
 @given(strategies.binary())
@@ -43,10 +44,6 @@
     def getcode(self):
         return self.__code
 
-class FakeCode:
-    def __init__(self, co_filename):
-        self.co_filename = co_filename
-
 def check_add_breakpoint(input, curfilename=None,
                          expected_funcname=None,
                          expected_fileline=None,
@@ -60,7 +57,7 @@
             messages.append((cmd, arg1, arg2, arg3, extra))
         def my_cur_frame():
             assert curfilename is not None
-            return FakeFrame(FakeCode(curfilename))
+            return FakeFrame(FakeCode(co_filename=curfilename))
         revdb.send_answer = got_message
         reverse_debugging.fetch_cur_frame = my_cur_frame
         add_breakpoint(input, 5)

From pypy.commits at gmail.com  Wed Aug 31 11:10:29 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 31 Aug 2016 08:10:29 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update,
 regenerate download links for release
Message-ID: <57c6f365.82cbc20a.81584.1d62@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r781:d2a617895143
Date: 2016-08-31 18:09 +0300
http://bitbucket.org/pypy/pypy.org/changeset/d2a617895143/

Log:	update, regenerate download links for release

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,7 +74,7 @@
 performance improvements.</p>
 <p>We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:</p>
 <ul class="simple">
-<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.3.1</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.3.0.html">what's new in PyPy2.7?</a>  and release note for <a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.3.1.html">PyPy2.7-v5.3.1</a>)</li>
+<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.4.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.0.html">what's new in PyPy2.7?</a> )</li>
 <li>the Python3.3 compatible release &#8212; <strong>PyPy3.3 v5.2-alpha</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy3.3-v5.2-alpha1.html">what's new in PyPy3.3?</a>).</li>
 <li>the Python2.7 Software Transactional Memory special release &#8212; <strong>PyPy-STM 2.5.1</strong> (Linux x86-64 only)</li>
 </ul>
@@ -114,22 +114,22 @@
 <li>or <a class="reference internal" href="#translate">translate</a> your own PyPy.</li>
 </ul>
 </div>
-<div class="section" id="python2-7-compatible-pypy-5-3-1">
-<h1>Python2.7 compatible PyPy 5.3.1</h1>
+<div class="section" id="python2-7-compatible-pypy-5-4-0">
+<h1>Python2.7 compatible PyPy 5.4.0</h1>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
 <li>FreeBSD 9.2 x86 64 bit <em>(hopefully availabe soon)</em> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
 installer <a class="reference external" href="http://www.microsoft.com/en-us/download/details.aspx?id=5582">vcredist_x86.exe</a>.)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0+-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0+-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-s390x.tar.bz2">s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-s390x.tar.bz2">s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads">All our downloads,</a> including previous versions.  We also have a
 <a class="reference external" href="http://buildbot.pypy.org/mirror/">mirror</a>, but please use only if you have troubles accessing the links above</li>
 </ul>
@@ -198,7 +198,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in <tt class="docutils literal">/opt</tt>, and
 if you want, put a symlink from somewhere like
-<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy2-5.3.1/bin/pypy</span></tt>.  Do
+<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy2-5.4.0/bin/pypy</span></tt>.  Do
 not move or copy the executable <tt class="docutils literal">pypy</tt> outside the tree &ndash; put
 a symlink to it, otherwise it will not find its libraries.</p>
 </div>
@@ -263,7 +263,7 @@
 <li><p class="first">Get the source code.  The following packages contain the source at
 the same revision as the above binaries:</p>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-src.tar.bz2">pypy2-v5.3.1-src.tar.bz2</a> (sources)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.tar.bz2">pypy2-v5.4.0-src.tar.bz2</a> (sources)</li>
 </ul>
 <p>Or you can checkout the current trunk using <a class="reference external" href="https://www.mercurial-scm.org/">Mercurial</a> (the trunk
 usually works and is of course more up-to-date):</p>
@@ -403,31 +403,20 @@
 <div class="section" id="checksums">
 <h1>Checksums</h1>
 <p>Here are the checksums for each of the downloads</p>
-<p>pypy2.7-v5.3 md5:</p>
+<p>pypy2.7-v5.4.0 md5:</p>
 <pre class="literal-block">
-05078bcdd797a025223d5905e2a12332  pypy2-v5.3.0-linux32.tar.bz2
-7d01e12eaca473258801faebc2db12d8  pypy2-v5.3.0-linux64.tar.bz2
-bf1640973865c5ca1bc88e299455cbcc  pypy2-v5.3.0-linux-armel.tar.bz2
-2aa5941a05d46427293d48d67d079df5  pypy2-v5.3.0-linux-armhf-raring.tar.bz2
-9c2cc832ba15fd4a08ba7e676226f406  pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
-21a346cca4e8e6897381a0e647a86d68  pypy2-v5.3.0-osx64.tar.bz2
-c39a578078ab3145d2a584cacf4c164c  pypy2-v5.3.0-s390x.tar.bz2
-45ce35a438ed8ae1539cc05407d43965  pypy2-v5.3.0-src.tar.bz2
-24add66f18ab2213c9e44af0ada61085  pypy2-v5.3.0-src.zip
-f6197adf58bfa32bcb18451289da1c7c  pypy2-v5.3.0-win32.zip
-</pre>
-<p>pypy2.7-v5.3.1 md5:</p>
-<pre class="literal-block">
-0ff0e50e9595448d882fe94ab8667993  pypy2-v5.3.1-linux32.tar.bz2
-41979b51bd5d8f9d6475b6478cf38992  pypy2-v5.3.1-linux64.tar.bz2
-0f929b98566b154473a50820a3a6cbcf  pypy2-v5.3.1-linux-armel.tar.bz2
-06ff729d3e30a9787ede69f327534d13  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
-be714716068e0e0c215e897f7c45ab34  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
-ae7c15c8b831847f359f673e3461b4e6  pypy2-v5.3.1-osx64.tar.bz2
-a74f104c25aeb69d2eb2cdce01a2cb02  pypy2-v5.3.1-s390x.tar.bz2
-2ebc87d24018c60cbf339de039dfecb0  pypy2-v5.3.1-src.tar.bz2
-a1bbafb57c26de5aae8700cd7667a86b  pypy2-v5.3.1-src.zip
-38e7d4dd33ea636cc7faebdd94ef6cb4  pypy2-v5.3.1-win32.zip
+50ea504e66f4d9297f5228d7a3b026ec  pypy2-v5.4.0-linux-armel.tar.bz2
+e838ba554bc53c793f23c378a898fa0f  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
+b1b9b755631ef85d400d7690ece50210  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
+df7180d5070ac19a234fc6c39b88f420  pypy2-v5.4.0-linux32.tar.bz2
+5e228ba05b6eaa0af37321fd3f425891  pypy2-v5.4.0-linux64.tar.bz2
+b32d4c97275901665945f1f2813b6f26  pypy2-v5.4.0-osx64.tar.bz2
+1d32ef8036a9fe718f397813bd070be8  pypy2-v5.4.0-ppc64.tar.bz2
+d8abb09416b4370ea40c51a710d12b18  pypy2-v5.4.0-ppc64le.tar.bz2
+b560c2811a3089f22b21db9beea7f273  pypy2-v5.4.0-s390x.tar.bz2
+c806bea7ecbb999fffeea3a06e6462e8  pypy2-v5.4.0-src.tar.bz2
+26c2ab1c891651eb620dbde499088c1f  pypy2-v5.4.0-src.zip
+bd25b15c0d6c0f7c7f6fa75f1da35014  pypy2-v5.4.0-win32.zip
 </pre>
 <p>pypy3.3-v5.2-alpha md5:</p>
 <pre class="literal-block">
@@ -446,31 +435,20 @@
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 </pre>
-<p>pypy2.7-5.3 sha1:</p>
+<p>pypy2.7-5.4.0 sha1:</p>
 <pre class="literal-block">
-401066f82c8a26dfb1e3421ae4b117655b55ee8d  pypy2-v5.3.0-linux32.tar.bz2
-939a49319ed8e25ecb9f646ba50f0618eb52c99b  pypy2-v5.3.0-linux64.tar.bz2
-00fe6e6d672c65d9096b45a708a1be95bca412c4  pypy2-v5.3.0-linux-armel.tar.bz2
-08fd47ffdf3bc9b7409147e9a0a576e0d577d735  pypy2-v5.3.0-linux-armhf-raring.tar.bz2
-1572762c5b76a6efda27110267f165bf9b78402d  pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
-6d6219b7d42b6f31118d2ace2280932dd52d4d9d  pypy2-v5.3.0-osx64.tar.bz2
-1d4f5f547d798294755fc9e14e2b6490e8a5d194  pypy2-v5.3.0-s390x.tar.bz2
-7bdb1cfc604192fc2a39023588d648a30975f0e4  pypy2-v5.3.0-src.tar.bz2
-18a81c46b3d0ecf7e186c19a7d302986a5b15a83  pypy2-v5.3.0-src.zip
-076251ba3b44435dc11867dab00f392b058bdc7c  pypy2-v5.3.0-win32.zip
-</pre>
-<p>pypy2.7-5.3.1 sha1:</p>
-<pre class="literal-block">
-46dcd486ce2acbdf1815b29d70295ad305b667c5  pypy2-v5.3.1-linux32.tar.bz2
-a06eba349f6346dd6ab4c47f9dcccb0b42986478  pypy2-v5.3.1-linux64.tar.bz2
-60f33190bae1ac774f461b1da8f974ba1e8c0c24  pypy2-v5.3.1-linux-armel.tar.bz2
-3fd2fa66437ce72c179ea76522407cde74456e36  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
-3f7f2aea02e90c6f4a5da00588cd06fdb74aa406  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
-1d866608f21a465f58ec6f9003f837262f6f7b1a  pypy2-v5.3.1-osx64.tar.bz2
-08a31fe87ea99864780db86bdb7d7fb9029dc54c  pypy2-v5.3.1-s390x.tar.bz2
-3abd0c4d133fde7198bf81b15b7786e4e3de9f9f  pypy2-v5.3.1-src.tar.bz2
-fbfaba410f5031041907a28452cfd5e46b8a7336  pypy2-v5.3.1-src.zip
-2963d6b1a87f284dfc0336cbec4c37eeb1918f41  pypy2-v5.3.1-win32.zip
+c50062a83e4bb9fc59b76901c92e7bf1ecd0351f  pypy2-v5.4.0-linux-armel.tar.bz2
+f4ebad7a9a31dfa55b35cc01b0533ef8e31ab7c4  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
+c0becdcb7f44e09947afab9df759313ec94563ef  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
+63be7254bdecd4f3272bcc47f0da7f5db82435a0  pypy2-v5.4.0-linux32.tar.bz2
+b0e0405ca8f3b143e16122767eb5605d3388af0c  pypy2-v5.4.0-linux64.tar.bz2
+9c97f54d492886fcaae8611733bcc40a625c8245  pypy2-v5.4.0-osx64.tar.bz2
+4a263167bbc89447e5adc2ed687ed44798bbca08  pypy2-v5.4.0-ppc64.tar.bz2
+b3554db74a826fd8e86f1132e9c2cb2e49caac1c  pypy2-v5.4.0-ppc64le.tar.bz2
+165920a2d0eeda83e8808e7fce93f2a9db7f736a  pypy2-v5.4.0-s390x.tar.bz2
+95163f8f3c8e9e52e126fc1807d8d94e3d224aec  pypy2-v5.4.0-src.tar.bz2
+b26546821836cb4bfda0160d37d4dd31fd3aace8  pypy2-v5.4.0-src.zip
+5ec0ca235cc68b557770b8cf5e1e49bd7b1a0aad  pypy2-v5.4.0-win32.zip
 </pre>
 <p>pypy3.3-v5.2-alpha sha1:</p>
 <pre class="literal-block">
@@ -484,31 +462,20 @@
 4b31ab492716ea375dd090bbacdf3d7c2d483059  pypy3.3-v5.2.0-alpha1-src.tar.bz2
 d9f5b64f144ebec1a200156809fbbe04fdf7eb7e  pypy3.3-v5.2.0-alpha1-src.zip
 </pre>
-<p>pypy2.7-5.3 sha256:</p>
+<p>pypy2.7-5.4.0 sha256:</p>
 <pre class="literal-block">
-bd422fe9d0b7d525d1da3f32855b047bc39ba397d0cf708d8f4f96fe874424f2  pypy2-v5.3.0-linux32.tar.bz2
-ac336e8877ed676bf87a9a546d5926b6afc4679fa2d3fdf9f3ca56f28ec40588  pypy2-v5.3.0-linux64.tar.bz2
-81b6f589a947d7353bb69408c46d4833d6e9cb501f3c3f0c73bd28d0e3df69aa  pypy2-v5.3.0-linux-armel.tar.bz2
-bdb911a87e773a292334061b9c33b907f46d987e403fe94cc627a3b9b1c9cb19  pypy2-v5.3.0-linux-armhf-raring.tar.bz2
-87b3566b6bbb8bf31c2f0d72bf31d95142fdce004d987812336a59d788005bed  pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
-1b103bacbdcdbbc490660ec0c7b3d99d1ff1cfc2f13cd403db21c27f03d36a1d  pypy2-v5.3.0-osx64.tar.bz2
-1ccc0ce08dd55d188d3971331020a1c82e917e418846d2c2c07a225733d85b1e  pypy2-v5.3.0-s390x.tar.bz2
-4142eb8f403810bc88a4911792bb5a502e152df95806e33e69050c828cd160d5  pypy2-v5.3.0-src.tar.bz2
-09914006c2c5c394bb6f847f6eb9c9322737c7238e7ca482c5a12c9e2ef76a58  pypy2-v5.3.0-src.zip
-32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20  pypy2-v5.3.0-win32.zip
-</pre>
-<p>pypy2.7-5.3.1 sha256:</p>
-<pre class="literal-block">
-da69f4280b288e524387103eaa3eb4d036965724c3e546da27135c15a77bd2eb  pypy2-v5.3.1-linux32.tar.bz2
-6d0e8b14875b76b1e77f06a2ee3f1fb5015a645a951ba7a7586289344d4d9c22  pypy2-v5.3.1-linux64.tar.bz2
-0425f2022c35ef7f0bb3d2b854c5bcbe500b1aba511a0d83581ba6c784913961  pypy2-v5.3.1-linux-armel.tar.bz2
-b4859496099bde4b17c1e56cc5749dcdcd25b4c68fde1d2ea426de84130e84cc  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
-5c93eb3c54fbb2c7d7332f775a096671512e590565e6051196bbc5039c5033b5  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
-7a242d7373b4f18c7f5fe6c2fe6f15e2a405d9adf1f4f934c89b875e60ac5def  pypy2-v5.3.1-osx64.tar.bz2
-61262f0727ee04b225761b59ce270a64fae9b986d22405a93340f05d0d5c0e0e  pypy2-v5.3.1-s390x.tar.bz2
-31a52bab584abf3a0f0defd1bf9a29131dab08df43885e7eeddfc7dc9b71836e  pypy2-v5.3.1-src.tar.bz2
-7eab4a8435583750088001d88371cd0314999b67f26f32ab9d006802f0beec2e  pypy2-v5.3.1-src.zip
-d83477e2c5f032ebd8c7f47afce03dc8adbeb41a3c74f7db50d9de317dcf3a4a  pypy2-v5.3.1-win32.zip
+04509044f21bb41ee6d3fafcf637fc0c586c248d4cdae6ac3357606a7b660fdb  pypy2-v5.4.0-linux-armel.tar.bz2
+95c690bcae6771ebce6cf06c7c2842e0662e007e35162afc963337aa597b471a  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
+839b08db89b7e20cb670b8cf02596e033ea0b76fb8336af7bedfbb04b6b502da  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
+ce581270464b14cdecd13dedb9bd7bf98232f767ac4ac282229a405d8e807af1  pypy2-v5.4.0-linux32.tar.bz2
+bdfea513d59dcd580970cb6f79f3a250d00191fd46b68133d5327e924ca845f8  pypy2-v5.4.0-linux64.tar.bz2
+3adf21c2bf3432759c99123f21240d71a72aba81d73129e48ef912c34631b723  pypy2-v5.4.0-osx64.tar.bz2
+dc09a057264dafb7e4bceca57b6a6ba3b0a5273e125a9b29da32b8439f980270  pypy2-v5.4.0-ppc64.tar.bz2
+4feb0711e7c235b247f8ea0b22e8a676f89e8831488b7a4e9c7f3a6943d07052  pypy2-v5.4.0-ppc64le.tar.bz2
+6bceb2760b1c7d6105d20207102862160ddddfd9b1a2707b3a8d866ac29e08d3  pypy2-v5.4.0-s390x.tar.bz2
+d9568ebe9a14d0eaefde887d78f3cba63d665e95c0d234bb583932341f55a655  pypy2-v5.4.0-src.tar.bz2
+3c165676be8df3b482727438836a9a240ea641392ddd60593f825e1d50029022  pypy2-v5.4.0-src.zip
+442c0a917781b6155bf78d2648f1ccd9a36c321926a043f83efcea22a99960b4  pypy2-v5.4.0-win32.zip
 </pre>
 <p>pypy3.3-v5.2-alpha sha256:</p>
 <pre class="literal-block">
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -14,13 +14,12 @@
 
 We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
 
-* the Python2.7 compatible release &#8212; **PyPy2.7 v5.3.1** &#8212; (`what's new in PyPy2.7?`_  and release note for `PyPy2.7-v5.3.1`_)
+* the Python2.7 compatible release &#8212; **PyPy2.7 v5.4.0** &#8212; (`what's new in PyPy2.7?`_ )
 * the Python3.3 compatible release &#8212; **PyPy3.3 v5.2-alpha** &#8212; (`what's new in PyPy3.3?`_).
 
 * the Python2.7 Software Transactional Memory special release &#8212; **PyPy-STM 2.5.1** (Linux x86-64 only)
 
-.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-pypy2.7-v5.3.0.html
-.. _PyPy2.7-v5.3.1: http://doc.pypy.org/en/latest/release-pypy2.7-v5.3.1.html
+.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.0.html
 .. _what's new in PyPy3.3?: http://doc.pypy.org/en/latest/release-pypy3.3-v5.2-alpha1.html
 
 
@@ -76,7 +75,7 @@
 .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux
 
 
-Python2.7 compatible PyPy 5.3.1
+Python2.7 compatible PyPy 5.4.0
 -----------------------------------
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
@@ -95,18 +94,18 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0+-ppc64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0+-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux32.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armhf-raring.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armel.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-ppc64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-ppc64le.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-s390x.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.zip
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
@@ -206,7 +205,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in ``/opt``, and
 if you want, put a symlink from somewhere like
-``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.3.1/bin/pypy``.  Do
+``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.4.0/bin/pypy``.  Do
 not move or copy the executable ``pypy`` outside the tree --- put
 a symlink to it, otherwise it will not find its libraries.
 
@@ -292,9 +291,9 @@
 1. Get the source code.  The following packages contain the source at
    the same revision as the above binaries:
 
-   * `pypy2-v5.3.1-src.tar.bz2`__ (sources)
+   * `pypy2-v5.4.0-src.tar.bz2`__ (sources)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.tar.bz2
 
    Or you can checkout the current trunk using Mercurial_ (the trunk
    usually works and is of course more up-to-date)::

From pypy.commits at gmail.com  Wed Aug 31 11:13:14 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 Aug 2016 08:13:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Backport memoryview changes from 3.5
Message-ID: <57c6f40a.915c1c0a.f8309.5e4d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86782:d9e0a4feb77e
Date: 2016-08-31 16:12 +0100
http://bitbucket.org/pypy/pypy/changeset/d9e0a4feb77e/

Log:	Backport memoryview changes from 3.5

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -636,6 +636,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -10,27 +10,73 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
+from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
 from pypy.objspace.std.bytesobject import getbytevalue
-from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
+from rpython.rlib.unroll import unrolling_iterable
+
+MEMORYVIEW_MAX_DIM = 64
+MEMORYVIEW_SCALAR   = 0x0001
+MEMORYVIEW_C        = 0x0002
+MEMORYVIEW_FORTRAN  = 0x0004
+MEMORYVIEW_SCALAR   = 0x0008
+MEMORYVIEW_PIL      = 0x0010
 
 
 class W_MemoryView(W_Root):
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
-    _immutable_fields_ = ['format', 'itemsize']
 
-    def __init__(self, buf, format='B', itemsize=1):
+    def __init__(self, buf, format=None, itemsize=1, ndim=-1,
+                 shape=None, strides=None, suboffsets=None):
         assert isinstance(buf, Buffer)
         self.buf = buf
         self._hash = -1
+        # private copies of format, shape, itemsize, ... on this class
         self.format = format
         self.itemsize = itemsize
+        self.shape = shape
+        self.strides = strides
+        self.suboffsets = suboffsets
+        self.ndim = ndim
+        self.flags = 0
+        self.length = -1
+        self._init_flags()
+
+    # several fields are "overwritten" by the memory view (shape, strides, ...)
+    # thus use only those getter fields instead of directly accessing the fields
+    def getndim(self):
+        if self.ndim == -1:
+            return self.buf.getndim()
+        return self.ndim
+
+    def getshape(self):
+        if self.shape is None:
+            return self.buf.getshape()
+        return self.shape
+
+    def getstrides(self):
+        if self.strides is None:
+            return self.buf.getstrides()
+        return self.strides
+
+    def getitemsize(self):
+        return self.itemsize
+
+    # memoryview needs to modify the field 'format', to prevent the modification
+    # of the buffer, we save the new format here!
+    def getformat(self):
+        if self.format is None:
+            return self.buf.getformat()
+        return self.format
+
+    def setformat(self, value):
+        self.format = value
 
     def buffer_w_ex(self, space, flags):
         self._check_released(space)
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf, self.format, self.itemsize
+        return self.buf, self.getformat(), self.itemsize
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
@@ -63,10 +109,45 @@
     descr_ne = _make_descr__cmp('ne')
 
     def as_str(self):
+        return ''.join(self.copy_buffer())
+
+    def copy_buffer(self):
         buf = self.buf
-        return buf.as_str()
+        n_bytes = buf.getlength()
+        data = []
+        self._copy_rec(0, data, 0)
+        return data
+
+    def _copy_rec(self, idim, data, off):
+        shapes = self.getshape()
+        shape = shapes[idim]
+        strides = self.getstrides()
+
+        if self.getndim()-1 == idim:
+            self._copy_base(data,off)
+            return
+
+        # TODO add a test that has at least 2 dims
+        for i in range(shape):
+            self._copy_rec(idim+1,data,off)
+            off += strides[idim]
+
+    def _copy_base(self, data, off):
+        shapes = self.getshape()
+        step = shapes[0]
+        strides = self.getstrides()
+        itemsize = self.getitemsize()
+        for i in range(step):
+            bytes = self.buf.getslice(off, off+itemsize, 1, itemsize)
+            data.append(bytes)
+            off += strides[0]
+            # do notcopy data if the sub buffer is out of bounds
+            if off >= self.buf.getlength():
+                break
 
     def getlength(self):
+        if self.length != -1:
+            return self.length // self.itemsize
         return self.buf.getlength() // self.itemsize
 
     def descr_tobytes(self, space):
@@ -75,43 +156,184 @@
 
     def descr_tolist(self, space):
         self._check_released(space)
+
+        buf = self.buf
+        dim = self.getndim()
+        fmt = self.getformat()
+        if dim == 0:
+            raise NotImplementedError
+        elif dim == 1:
+            itemsize = self.getitemsize()
+            return self._tolist(space, buf, buf.getlength() // itemsize, fmt)
+        else:
+            return self._tolist_rec(space, buf, 0, 0, fmt)
+
+    def _tolist(self, space, buf, count, fmt):
         # TODO: this probably isn't very fast
-        fmtiter = UnpackFormatIterator(space, self.buf)
-        fmtiter.interpret(self.format * self.getlength())
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(fmt * count)
         return space.newlist(fmtiter.result_w)
 
+    def _tolist_rec(self, space, buf, start, idim, fmt):
+        strides = self.getstrides()
+        shape = self.getshape()
+        #
+        dim = idim+1
+        stride = strides[idim]
+        itemsize = self.getitemsize()
+        dimshape = shape[idim]
+        #
+        if dim >= self.getndim():
+            bytecount = (stride * dimshape)
+            count = bytecount // itemsize
+            return self._tolist(space, buf, count, fmt)
+        items = [None] * dimshape
+
+        for i in range(dimshape):
+            item = self._tolist_rec(space, SubBuffer(buf, start, stride), start, idim+1, fmt)
+            items[i] = item
+            start += stride
+
+        return space.newlist(items)
+
+
+    def _start_from_tuple(self, space, w_tuple):
+        from pypy.objspace.std.tupleobject import W_TupleObject
+        start = 0
+
+        view = self.buf
+        length = space.len_w(w_tuple)
+        dim = view.getndim()
+        dim = 0
+        assert isinstance(w_tuple, W_TupleObject)
+        while dim < length:
+            w_obj = w_tuple.getitem(space, dim)
+            index = w_obj.int_w(space)
+            start = self.lookup_dimension(space, start, dim, index)
+            dim += 1
+        return start
+
+    def lookup_dimension(self, space, start, dim, index):
+        view = self.buf
+        shape = view.getshape()
+        strides = view.getstrides()
+        nitems = shape[dim]
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim+1)
+        start += strides[dim] * index
+        # TODO suboffsets?
+        return start
+
+    def _getitem_tuple_indexed(self, space, w_index):
+        view = self.buf
+
+        fmt = view.getformat() # TODO adjust format?
+
+        length = space.len_w(w_index)
+        ndim = view.getndim()
+        if length < ndim:
+            raise OperationError(space.w_NotImplementedError, \
+                    space.wrap("sub-views are not implemented"))
+
+        if length > ndim:
+            raise oefmt(space.w_TypeError, \
+                    "cannot index %d-dimension view with %d-element tuple",
+                    length, ndim)
+
+        start = self._start_from_tuple(space, w_index)
+
+        buf = SubBuffer(self.buf, start, view.getitemsize())
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(fmt)
+        return fmtiter.result_w[0]
+
+
     def descr_getitem(self, space, w_index):
         self._check_released(space)
+
+        if space.isinstance_w(w_index, space.w_tuple):
+            return self._getitem_tuple_indexed(space, w_index)
+
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
-        itemsize = self.itemsize
+        itemsize = self.getitemsize()
+        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
             if itemsize == 1:
                 ch = self.buf.getitem(start)
                 return space.newint(ord(ch))
             else:
                 # TODO: this probably isn't very fast
-                buf = SubBuffer(self.buf, start * itemsize, itemsize)
+                buf = SubBuffer(self.buf, start, itemsize)
                 fmtiter = UnpackFormatIterator(space, buf)
                 fmtiter.interpret(self.format)
                 return fmtiter.result_w[0]
         elif step == 1:
-            buf = SubBuffer(self.buf, start * itemsize, size * itemsize)
-            return W_MemoryView(buf, self.format, itemsize)
+            mv = W_MemoryView.copy(self)
+            mv.slice(start, stop, step, size)
+            mv._init_flags()
+            return mv
         else:
-            # XXX needs to return a W_MemoryView with a NonContiguousSubBuffer
-            # maybe?  Need to check the cpyext requirements for that
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX extended slicing")
+            mv = W_MemoryView.copy(self)
+            mv.slice(start, stop, step, size)
+            mv.length = mv.bytecount_from_shape()
+            mv._init_flags()
+            return mv
+
+    def slice(self, start, stop, step, size):
+        # modifies the buffer, shape and stride to allow step to be > 1
+        # NOTE that start, stop & size are already byte offsets/count
+        # TODO subbuffer
+        strides = self.getstrides()[:]
+        shape = self.getshape()[:]
+        itemsize = self.getitemsize()
+        dim = 0
+        self.buf = SubBuffer(self.buf, strides[dim] * (start//itemsize), size*step)
+        shape[dim] = size
+        strides[dim] = strides[dim] * step
+        self.strides = strides
+        self.shape = shape
+
+    def bytecount_from_shape(self):
+        dim = self.getndim()
+        shape = self.getshape()
+        length = 1
+        for i in range(dim):
+            length *= shape[i]
+        return length * self.getitemsize()
+
+    @staticmethod
+    def copy(view, buf=None):
+        # TODO suboffsets
+        if buf == None:
+            buf = view.buf
+        return W_MemoryView(buf, view.getformat(), view.getitemsize(),
+                            view.getndim(), view.getshape()[:], view.getstrides()[:])
+
+    def _apply_itemsize(self, space, start, size, itemsize):
+        if itemsize > 1:
+            start *= itemsize
+            size *= itemsize
+
+        stop  = start + size
+        # start & stop are now byte offset, thus use self.buf.getlength()
+        if stop > self.buf.getlength():
+            raise oefmt(space.w_IndexError, 'index out of range')
+
+        return start, stop, size
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         if space.isinstance_w(w_index, space.w_tuple):
-            raise oefmt(space.w_NotImplementedError, "XXX tuple setitem")
+            raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.itemsize
+        itemsize = self.getitemsize()
+        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
             if itemsize == 1:
                 ch = getbytevalue(space, w_obj)
@@ -125,16 +347,41 @@
                     raise oefmt(space.w_TypeError,
                                 "memoryview: invalid type for format '%s'",
                                 self.format)
-                self.buf.setslice(start * itemsize, fmtiter.result.build())
+                self.buf.setslice(start, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-            if value.getlength() != size * self.itemsize:
+            if value.getlength() != size:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
-            self.buf.setslice(start * itemsize, value.as_str())
+            self.buf.setslice(start, value.as_str())
         else:
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX extended slicing")
+            if self.getndim() != 1:
+                raise oefmt(space.w_NotImplementedError,
+                        "memoryview slice assignments are currently "
+                        "restricted to ndim = 1")
+            # this is the case of a one dimensional copy!
+            # NOTE we could maybe make use of copy_base, but currently we do not
+            itemsize = self.getitemsize()
+            data = []
+            src = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+            dst_strides = self.getstrides()
+            dim = 0
+            dst = SubBuffer(self.buf, start, size)
+            src_stride0 = dst_strides[dim]
+
+            off = 0
+            src_shape0 = size // itemsize
+            src_stride0 = src.getstrides()[0]
+            if isinstance(w_obj, W_MemoryView):
+                src_stride0 = w_obj.getstrides()[0]
+            for i in range(src_shape0):
+                data.append(src.getslice(off,off+itemsize,1,itemsize))
+                off += src_stride0
+            off = 0
+            dst_stride0 = self.getstrides()[0] * step
+            for dataslice in data:
+                dst.setslice(off, dataslice)
+                off += dst_stride0
 
     def descr_len(self, space):
         self._check_released(space)
@@ -142,11 +389,11 @@
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap(self.format)
+        return space.wrap(self.getformat())
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.newint(self.itemsize)
+        return space.wrap(self.itemsize)
 
     def w_get_ndim(self, space):
         self._check_released(space)
@@ -158,11 +405,11 @@
 
     def w_get_shape(self, space):
         self._check_released(space)
-        return space.newtuple([space.newint(self.getlength())])
+        return space.newtuple([space.wrap(x) for x in self.getshape()])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.newint(self.itemsize)])
+        return space.newtuple([space.wrap(x) for x in self.getstrides()])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)
@@ -240,14 +487,178 @@
             size = rffi.sizeof(rffi.VOIDP)
         return size
 
+    def _zero_in_shape(self):
+        # this method could be moved to the class Buffer
+        buf = self.buf
+        shape = buf.getshape()
+        for i in range(buf.getndim()):
+            if shape[i] == 0:
+                return True
+        return False
+
     def descr_cast(self, space, w_format, w_shape=None):
         self._check_released(space)
-        if not space.is_none(w_shape):
-            raise oefmt(space.w_NotImplementedError,
-                        "XXX cast() with a shape")
+
+        if not space.isinstance_w(w_format, space.w_unicode):
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: format argument must be a string"))
+
         fmt = space.str_w(w_format)
-        newitemsize = self.get_native_fmtchar(fmt)
-        return W_MemoryView(self.buf, fmt, newitemsize)
+        buf = self.buf
+        ndim = 1
+
+        if not memory_view_c_contiguous(space, self.flags):
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: casts are restricted" \
+                               " to C-contiguous views"))
+
+        if (w_shape or buf.getndim() != 1) and self._zero_in_shape():
+            raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: cannot casts view with" \
+                               " zeros in shape or strides"))
+
+        itemsize = self.get_native_fmtchar(fmt)
+        if w_shape:
+            if not (space.isinstance_w(w_shape, space.w_list) or space.isinstance_w(w_shape, space.w_tuple)):
+                raise oefmt(space.w_TypeError, "expected list or tuple got %T", w_shape)
+            ndim = space.len_w(w_shape)
+            if ndim > MEMORYVIEW_MAX_DIM:
+                raise oefmt(space.w_ValueError, \
+                        "memoryview: number of dimensions must not exceed %d",
+                        ndim)
+            # yes access ndim as field
+            if self.ndim > 1 and buf.getndim() != 1:
+                raise OperationError(space.w_TypeError, \
+                    space.wrap("memoryview: cast must be 1D -> ND or ND -> 1D"))
+
+        mv = W_MemoryView(buf, self.format, self.itemsize)
+        origfmt = mv.getformat()
+        mv._cast_to_1D(space, origfmt, fmt, itemsize)
+        if w_shape:
+            fview = space.fixedview(w_shape)
+            shape = [space.int_w(w_obj) for w_obj in fview]
+            mv._cast_to_ND(space, shape, ndim)
+        return mv
+
+    def _init_flags(self):
+        buf = self.buf
+        ndim = buf.getndim()
+        flags = 0
+        if ndim == 0:
+            flags |= MEMORYVIEW_SCALAR | MEMORYVIEW_C | MEMORYVIEW_FORTRAN
+        if ndim == 1:
+            shape = buf.getshape()
+            strides = buf.getstrides()
+            if len(shape) > 0 and shape[0] == 1 and \
+               len(strides) > 0 and strides[0] == buf.getitemsize():
+                flags |= MEMORYVIEW_C | MEMORYVIEW_SCALAR
+        # TODO for now?
+        flags |= MEMORYVIEW_C
+        # TODO if buf.is_contiguous('C'):
+        # TODO     flags |= MEMORYVIEW_C
+        # TODO elif buf.is_contiguous('F'):
+        # TODO     flags |= MEMORYVIEW_FORTRAN
+
+        # TODO missing suboffsets
+
+        self.flags = flags
+
+    def _cast_to_1D(self, space, origfmt, fmt, itemsize):
+        buf = self.buf
+        if itemsize < 0:
+            raise oefmt(space.w_ValueError, "memoryview: destination" \
+                    " format must be a native single character format prefixed" \
+                    " with an optional '@'")
+
+        if self.get_native_fmtchar(origfmt) < 0 or \
+           (not is_byte_format(fmt) and not is_byte_format(origfmt)):
+            raise oefmt(space.w_TypeError,
+                    "memoryview: cannot cast between" \
+                    " two non-byte formats")
+
+        if buf.getlength() % itemsize != 0:
+            raise oefmt(space.w_TypeError,
+                    "memoryview: length is not a multiple of itemsize")
+
+        newfmt = self.get_native_fmtstr(fmt)
+        if not newfmt:
+            raise oefmt(space.w_RuntimeError,
+                    "memoryview: internal error")
+        self.format = newfmt
+        self.itemsize = itemsize
+        self.ndim = 1
+        self.shape = [buf.getlength() // itemsize]
+        self.strides = [itemsize]
+        # XX suboffsets
+
+        self._init_flags()
+
+    def get_native_fmtstr(self, fmt):
+        lenfmt = len(fmt)
+        nat = False
+        if lenfmt == 0:
+            return None
+        elif lenfmt == 1:
+            format = fmt[0] # fine!
+        elif lenfmt == 2:
+            if fmt[0] == '@':
+                nat = True
+                format = fmt[1]
+            else:
+                return None
+        else:
+            return None
+
+        chars = ['c','b','B','h','H','i','I','l','L','q',
+                 'Q','n','N','f','d','?','P']
+        for c in unrolling_iterable(chars):
+            if c == format:
+                if nat: return '@'+c
+                else: return c
+
+        return None
+
+    def _cast_to_ND(self, space, shape, ndim):
+        buf = self.buf
+
+        self.ndim = ndim
+        length = self.itemsize
+        if ndim == 0:
+            self.shape = []
+            self.strides = []
+        else:
+            self.shape = shape
+            for i in range(ndim):
+                length *= shape[i]
+            self._init_strides_from_shape()
+
+        if length != self.buf.getlength():
+            raise OperationError(space.w_TypeError,
+                    space.wrap("memoryview: product(shape) * itemsize != buffer size"))
+
+        self._init_flags()
+
+    def _init_strides_from_shape(self):
+        shape = self.getshape()
+        s = [0] * len(shape)
+        self.strides = s
+        ndim = self.getndim()
+        s[ndim-1] = self.itemsize
+        i = ndim-2
+        while i >= 0:
+            s[i] = s[i+1] * shape[i+1]
+            i -= 1
+
+    def descr_hex(self, space):
+        from pypy.objspace.std.bytearrayobject import _array_to_hexstring
+        self._check_released(space)
+        return _array_to_hexstring(space, self.buf)
+
+def is_byte_format(char):
+    return char == 'b' or char == 'B' or char == 'c'
+
+def memory_view_c_contiguous(space, flags):
+    return flags & (space.BUF_CONTIG_RO|MEMORYVIEW_C) != 0
 
 W_MemoryView.typedef = TypeDef(
     "memoryview",
@@ -266,6 +677,7 @@
     __exit__    = interp2app(W_MemoryView.descr_exit),
     __weakref__ = make_weakref_descr(W_MemoryView),
     cast        = interp2app(W_MemoryView.descr_cast),
+    #hex         = interp2app(W_MemoryView.descr_hex),  # 3.5
     tobytes     = interp2app(W_MemoryView.descr_tobytes),
     tolist      = interp2app(W_MemoryView.descr_tolist),
     release     = interp2app(W_MemoryView.descr_release),
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -1,3 +1,11 @@
+import py
+import struct
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.typedef import TypeDef
+from rpython.rlib.buffer import Buffer
+from pypy.conftest import option
+
 class AppTestMemoryView:
     spaceconfig = dict(usemodules=['array'])
 
@@ -16,6 +24,7 @@
         w = v[1:234]
         assert isinstance(w, memoryview)
         assert len(w) == 2
+        exc = raises(TypeError, "memoryview('foobar')")
 
     def test_rw(self):
         data = bytearray(b'abcefg')
@@ -28,7 +37,7 @@
         v[0:3] = v[2:5]
         assert data == bytearray(eval("b'23f3fg'"))
         exc = raises(ValueError, "v[2:3] = b'spam'")
-        assert str(exc.value) == "cannot modify size of memoryview object"
+        #assert str(exc.value) == "cannot modify size of memoryview object"
 
     def test_extended_slice(self):
         data = bytearray(b'abcefg')
@@ -37,7 +46,7 @@
         assert len(w) == 1
         assert list(w) == [97]
         v[::2] = b'ABC'
-        assert data == bytearray(b'AbBeCg')
+        assert data == bytearray(eval("b'AbBeCg'"))
 
     def test_memoryview_attrs(self):
         v = memoryview(b"a"*100)
@@ -213,4 +222,159 @@
         data = bytearray(b'abcdefghij')
         m3 = memoryview(data).cast('h')
         m3[1:5:2] = memoryview(b"xyXY").cast('h')
-        assert data == bytearray(b'abxyefXYij')
+        assert data == bytearray(eval("b'abxyefXYij'"))
+
+class MockBuffer(Buffer):
+    def __init__(self, space, w_arr, w_dim, w_fmt, \
+                 w_itemsize, w_strides, w_shape):
+        self.space = space
+        self.w_arr = w_arr
+        self.arr = []
+        self.ndim = space.int_w(w_dim)
+        self.format = space.str_w(w_fmt)
+        self.itemsize = space.int_w(w_itemsize)
+        self.strides = []
+        for w_i in w_strides.getitems_unroll():
+            self.strides.append(space.int_w(w_i))
+        self.shape = []
+        for w_i in w_shape.getitems_unroll():
+            self.shape.append(space.int_w(w_i))
+        self.readonly = True
+        self.shape.append(space.len_w(w_arr))
+        self.data = []
+        itemsize = 1
+        worklist = [(1,w_arr)]
+        while worklist:
+            dim, w_work = worklist.pop()
+            if space.isinstance_w(w_work, space.w_list):
+                for j, w_obj in enumerate(w_work.getitems_unroll()):
+                    worklist.insert(0, (dim+1, w_obj))
+                continue
+            byte = struct.pack(self.format, space.int_w(w_work))
+            for c in byte:
+                self.data.append(c)
+        self.data = ''.join(self.data)
+
+    def getformat(self):
+        return self.format
+
+    def getitem(self, index):
+        return self.data[index:index+1]
+
+    def getlength(self):
+        return len(self.data)
+
+    def getitemsize(self):
+        return self.itemsize
+
+    def getndim(self):
+        return self.ndim
+
+    def getstrides(self):
+        return self.strides
+
+    def getshape(self):
+        return self.shape
+
+    def is_contiguous(self, format):
+        return format == 'C'
+
+class W_MockArray(W_Root):
+    def __init__(self, w_list, w_dim, w_fmt, w_size, w_strides, w_shape):
+        self.w_list = w_list
+        self.w_dim = w_dim
+        self.w_fmt = w_fmt
+        self.w_size = w_size
+        self.w_strides = w_strides
+        self.w_shape = w_shape
+
+    @staticmethod
+    def descr_new(space, w_type, w_list, w_dim, w_fmt, \
+                         w_size, w_strides, w_shape):
+        return W_MockArray(w_list, w_dim, w_fmt, w_size, w_strides, w_shape)
+
+    def buffer_w(self, space, flags):
+        return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
+                          self.w_size, self.w_strides, self.w_shape)
+
+    def buffer_w_ex(self, space, flags):
+        return self.buffer_w(space, flags), space.str_w(self.w_fmt), space.int_w(self.w_size)
+
+W_MockArray.typedef = TypeDef("MockArray",
+    __new__ = interp2app(W_MockArray.descr_new),
+)
+
+class AppTestMemoryViewMockBuffer(object):
+    spaceconfig = dict(usemodules=[])
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("Impossible to run on appdirect")
+        cls.w_MockArray = cls.space.gettypefor(W_MockArray)
+
+    def test_tuple_indexing(self):
+        content = self.MockArray([[0,1,2,3], [4,5,6,7], [8,9,10,11]],
+                                 dim=2, fmt='B', size=1,
+                                 strides=[4,1], shape=[3,4])
+        view = memoryview(content)
+        assert view[0,0] == 0
+        assert view[2,0] == 8
+        assert view[2,3] == 11
+        assert view[-1,-1] == 11
+        assert view[-3,-4] == 0
+
+        raises(IndexError, "view.__getitem__((2**63-1,0))")
+        raises(TypeError, "view.__getitem__((0, 0, 0))")
+
+    def test_tuple_indexing_int(self):
+        content = self.MockArray([ [[1],[2],[3]], [[4],[5],[6]] ],
+                                 dim=3, fmt='i', size=4,
+                                 strides=[12,4,4], shape=[2,3,1])
+        view = memoryview(content)
+        assert view[0,0,0] == 1
+        assert view[-1,2,0] == 6
+
+    def test_cast_non_byte(self):
+        empty = self.MockArray([], dim=1, fmt='i', size=4, strides=[1], shape=[1])
+        view = memoryview(empty)
+        raises(TypeError, "view.cast('l')")
+        try:
+            view.cast('l')
+            assert False, "i -> l not possible. buffer must be byte format"
+        except TypeError:
+            pass
+
+    def test_cast_empty(self):
+        empty = self.MockArray([], dim=1, fmt='b', size=1, strides=[1], shape=[1])
+        view = memoryview(empty)
+        cview = view.cast('i')
+        assert cview.tobytes() == b''
+        assert cview.tolist() == []
+        assert view.format == 'b'
+        assert cview.format == 'i'
+        #
+        assert cview.cast('b').cast('q').cast('b').tolist() == []
+        #
+        assert cview.format == 'i'
+        raises(TypeError, "cview.cast('i')")
+
+    def test_cast_with_shape(self):
+        empty = self.MockArray([1,0,2,0,3,0],
+                    dim=1, fmt='h', size=2,
+                    strides=[8], shape=[6])
+        view = memoryview(empty)
+        byteview = view.cast('b')
+        assert byteview.tolist() == [1,0,0,0,2,0,0,0,3,0,0,0]
+        i32view = byteview.cast('i', shape=[1,3])
+        assert i32view.format == 'i'
+        assert i32view.itemsize == 4
+        assert i32view.tolist() == [[1,2,3]]
+        i32view = byteview.cast('i', shape=(1,3))
+        assert i32view.tolist() == [[1,2,3]]
+
+    def test_cast_bytes(self):
+        bytes = b"\x02\x00\x03\x00\x04\x00" \
+                b"\x05\x00\x06\x00\x07\x00"
+        view = memoryview(bytes)
+        v = view.cast('h', shape=(3,2))
+        assert v.tolist() == [[2,3],[4,5],[6,7]]
+        raises(TypeError, "view.cast('h', shape=(3,3))")

From pypy.commits at gmail.com  Wed Aug 31 11:15:36 2016
From: pypy.commits at gmail.com (vext01)
Date: Wed, 31 Aug 2016 08:15:36 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Fix more W^X.
Message-ID: <57c6f498.11331c0a.c3e17.59b5@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86783:0b20f17103f9
Date: 2016-08-31 16:15 +0100
http://bitbucket.org/pypy/pypy/changeset/0b20f17103f9/

Log:	Fix more W^X.

diff --git a/rpython/jit/backend/llsupport/gcreftracer.py b/rpython/jit/backend/llsupport/gcreftracer.py
--- a/rpython/jit/backend/llsupport/gcreftracer.py
+++ b/rpython/jit/backend/llsupport/gcreftracer.py
@@ -15,11 +15,15 @@
     obj = llmemory.cast_adr_to_ptr(obj_addr, lltype.Ptr(GCREFTRACER))
     i = 0
     length = obj.array_length
+    length_bytes = length * WORD
     addr = obj.array_base_addr
+    addr_p = rffi.cast(rffi.CCHARP, addr)
+    set_pages_writable(addr_p, length_bytes)
     while i < length:
         p = rffi.cast(llmemory.Address, addr + i * WORD)
         gc._trace_callback(callback, arg, p)
         i += 1
+    set_pages_executable(addr_p, length_bytes)
 lambda_gcrefs_trace = lambda: gcrefs_trace
 
 def make_framework_tracer(array_base_addr, gcrefs):
@@ -34,12 +38,13 @@
     tr.array_length = length
     i = 0
     array_base_addr_p = rffi.cast(rffi.CCHARP, array_base_addr)
-    rmmap.set_pages_writable(array_base_addr_p, length)
+    length_bytes = length * WORD
+    rmmap.set_pages_writable(array_base_addr_p, length_bytes)
     while i < length:
         p = rffi.cast(rffi.SIGNEDP, array_base_addr + i * WORD)
         p[0] = rffi.cast(lltype.Signed, gcrefs[i])
         i += 1
-    rmmap.set_pages_executable(array_base_addr_p, length)
+    rmmap.set_pages_executable(array_base_addr_p, length_bytes)
     llop.gc_writebarrier(lltype.Void, tr)
     # --no GC until here--
     return tr
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -746,6 +746,7 @@
     def set_pages_executable(addr, size):
         assert lltype.typeOf(addr) == rffi.CCHARP
         assert isinstance(size, int)
+        #assert size >= 0
         rv = mprotect(addr, size, PROT_EXEC | PROT_READ)
         if int(rv) < 0:
             from rpython.rlib import debug
@@ -754,6 +755,7 @@
     def set_pages_writable(addr, size):
         assert lltype.typeOf(addr) == rffi.CCHARP
         assert isinstance(size, int)
+        #assert size >= 0
         rv = mprotect(addr, size, PROT_WRITE | PROT_READ)
         if int(rv) < 0:
             from rpython.rlib import debug

From pypy.commits at gmail.com  Wed Aug 31 11:22:10 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 Aug 2016 08:22:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix shape of array buffers
Message-ID: <57c6f622.411d1c0a.e3fac.6651@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86784:00ac4fb4c63b
Date: 2016-08-31 16:21 +0100
http://bitbucket.org/pypy/pypy/changeset/00ac4fb4c63b/

Log:	Fix shape of array buffers

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -645,6 +645,9 @@
     def getndim(self):
         return 1
 
+    def getshape(self):
+        return [self.array.len]
+
     def getstrides(self):
         return [self.getitemsize()]
 
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -153,6 +153,7 @@
         assert m.format == 'i'
         assert m.itemsize == 4
         assert len(m) == 10
+        assert m.shape == (10,)
         assert len(m.tobytes()) == 40
         assert m[0] == 0
         m[0] = 1

From pypy.commits at gmail.com  Wed Aug 31 11:26:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 Aug 2016 08:26:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c6f735.6211c20a.f379a.2ddc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r86785:3d0c8318b98d
Date: 2016-08-31 16:26 +0100
http://bitbucket.org/pypy/pypy/changeset/3d0c8318b98d/

Log:	hg merge py3k

diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -29,10 +29,10 @@
 
     def descr_repr(self, space):
         if self.w_objtype is not None:
-            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+            objtype_name = u"<%s object>" % self.w_objtype.getname(space)
         else:
-            objtype_name = 'NULL'
-        return space.wrap("<super: <class '%s'>, %s>" % (
+            objtype_name = u'NULL'
+        return space.wrap(u"<super: <class '%s'>, %s>" % (
             self.w_starttype.getname(space), objtype_name))
 
     def get(self, space, w_obj, w_type=None):
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -645,6 +645,9 @@
     def getndim(self):
         return 1
 
+    def getshape(self):
+        return [self.array.len]
+
     def getstrides(self):
         return [self.getitemsize()]
 
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -37,7 +37,7 @@
         v[0:3] = v[2:5]
         assert data == bytearray(eval("b'23f3fg'"))
         exc = raises(ValueError, "v[2:3] = b'spam'")
-        assert str(exc.value) == "cannot modify size of memoryview object"
+        #assert str(exc.value) == "cannot modify size of memoryview object"
 
     def test_extended_slice(self):
         data = bytearray(b'abcefg')
@@ -153,6 +153,7 @@
         assert m.format == 'i'
         assert m.itemsize == 4
         assert len(m) == 10
+        assert m.shape == (10,)
         assert len(m.tobytes()) == 40
         assert m[0] == 0
         m[0] = 1

From pypy.commits at gmail.com  Wed Aug 31 12:25:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 09:25:11 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57c704e7.d41a1c0a.3b852.b83e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r782:db1924be1759
Date: 2016-08-31 18:25 +0200
http://bitbucket.org/pypy/pypy.org/changeset/db1924be1759/

Log:	update the values

diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -9,7 +9,7 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 38.6
+      value: 73.7
    });
   });
 </script>
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $30891 of $80000 (38.6%)
+   $58959 of $80000 (73.7%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $23541 left
+    we have $48187 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Wed Aug 31 12:53:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 09:53:29 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Translation fixes
Message-ID: <57c70b89.6211c20a.f379a.4f97@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86786:2c419984a223
Date: 2016-08-31 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/2c419984a223/

Log:	Translation fixes

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -206,7 +206,8 @@
     p = 0
     result = []
     while p < len(lnotab) - 1:
-        byte_incr = ord(lnotab[p])
+        byte_incr = ord(lnotab[p + 0])
+        line_incr = ord(lnotab[p + 1])
         if byte_incr:
             if lineno != lastlineno:
                 result.append((addr, lineno))
@@ -559,6 +560,7 @@
         # normalize co_filename, and assigns the {lineno: bkpt_num} dict
         # back over the original key, to avoid calling rabspath/rnormpath
         # again the next time
+        co_filename = rstring.assert_str0(co_filename)
         normfilename = rpath.rabspath(co_filename)
         normfilename = rpath.rnormpath(normfilename)
         linenos = dbstate.breakpoint_filelines.get(normfilename, None)
@@ -611,7 +613,8 @@
     # if it has no ':', it can be a valid identifier (which we
     # register as a function name), or a lineno
     original_name = name
-    if ':' not in name:
+    j = name.rfind(':')
+    if j < 0:
         try:
             lineno = int(name)
         except ValueError:
@@ -625,7 +628,6 @@
         filename = ''
     else:
         # if it has a ':', it must end in ':lineno'
-        j = name.rfind(':')
         try:
             lineno = int(name[j+1:])
         except ValueError:
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -179,7 +179,8 @@
     rawbytes = cache.wdict.get(w_x)
     if rawbytes is None:
         data = space.str_w(w_x)
-        if we_are_translated() and not rgc.can_move(data):
+        if (we_are_translated() and not rgc.can_move(data)
+                                and not rgc.must_split_gc_address_space()):
             lldata = llstr(data)
             data_start = (llmemory.cast_ptr_to_adr(lldata) +
                           rffi.offsetof(STR, 'chars') +

From pypy.commits at gmail.com  Wed Aug 31 13:18:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 10:18:45 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Copy some documentation about ``-X
 track-resources`` here as well
Message-ID: <57c71175.e2efc20a.1a16b.51ab@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r783:097e4f70c803
Date: 2016-08-31 19:18 +0200
http://bitbucket.org/pypy/pypy.org/changeset/097e4f70c803/

Log:	Copy some documentation about ``-X track-resources`` here as well

diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -116,6 +116,16 @@
 <div class="syntax python"><pre><span></span><span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s2">&quot;filename&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span><br/><span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">&quot;stuff&quot;</span><span class="p">)</span><br/><span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span><br/></pre></div>
 <p>or using the <tt class="docutils literal">with</tt> keyword</p>
 <div class="syntax python"><pre><span></span><span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">&quot;filename&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span><br/>    <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">&quot;stuff&quot;</span><span class="p">)</span><br/></pre></div>
+<p>The same problem&ndash;not closing your files&ndash;can also show up if your
+program opens a large number of files without closing them explicitly.
+In that case, you can easily hit the system limit on the number of file
+descriptors that are allowed to be opened at the same time.</p>
+<p>Since release 5.4, PyPy can be run with the command-line option <tt class="docutils literal"><span class="pre">-X</span>
+<span class="pre">track-resources</span></tt> (as in, <tt class="docutils literal">pypy <span class="pre">-X</span> <span class="pre">track-resources</span> myprogram.py</tt>).
+This produces a ResourceWarning when the GC closes a non-closed file or
+socket.  The traceback for the place where the file or socket was
+allocated is given as well, which aids finding places where <tt class="docutils literal">close()</tt>
+is missing.</p>
 <p>Similarly, remember that you must <tt class="docutils literal">close()</tt> a non-exhausted
 generator in order to have its pending <tt class="docutils literal">finally</tt> or <tt class="docutils literal">with</tt>
 clauses executed immediately:</p>
diff --git a/source/compat.txt b/source/compat.txt
--- a/source/compat.txt
+++ b/source/compat.txt
@@ -86,6 +86,18 @@
     with open("filename", "w") as f:
         f.write("stuff")
 
+The same problem---not closing your files---can also show up if your
+program opens a large number of files without closing them explicitly.
+In that case, you can easily hit the system limit on the number of file
+descriptors that are allowed to be opened at the same time.
+
+Since release 5.4, PyPy can be run with the command-line option ``-X
+track-resources`` (as in, ``pypy -X track-resources myprogram.py``).
+This produces a ResourceWarning when the GC closes a non-closed file or
+socket.  The traceback for the place where the file or socket was
+allocated is given as well, which aids finding places where ``close()``
+is missing.
+
 Similarly, remember that you must ``close()`` a non-exhausted
 generator in order to have its pending ``finally`` or ``with``
 clauses executed immediately:

From pypy.commits at gmail.com  Wed Aug 31 13:18:49 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 Aug 2016 10:18:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Remove debugging leftover
Message-ID: <57c71179.482cc20a.5879c.51fa@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86787:f06558622d28
Date: 2016-08-31 18:18 +0100
http://bitbucket.org/pypy/pypy/changeset/f06558622d28/

Log:	Remove debugging leftover

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -157,7 +157,6 @@
                 # poor man's x.decode('ascii', 'replace'), since it's not
                 # supported by RPython
                 if not we_are_translated():
-                    import pdb;pdb.set_trace()
                     print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
                 lst = []
                 for ch in x:

From pypy.commits at gmail.com  Wed Aug 31 13:58:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 10:58:32 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: user interface
Message-ID: <57c71ac8.8f8e1c0a.c5d55.93d3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86788:8aa212be6c45
Date: 2016-08-31 19:07 +0100
http://bitbucket.org/pypy/pypy/changeset/8aa212be6c45/

Log:	user interface

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -605,6 +605,7 @@
     # if it is empty, complain
     if not name:
         revdb.send_output("Empty breakpoint name\n")
+        revdb.send_change_breakpoint(i)
         return
     # if it is surrounded by < >, it is the name of a code object
     if name.startswith('<') and name.endswith('>'):
@@ -621,7 +622,7 @@
             if not valid_identifier(name):
                 revdb.send_output(
                     'Note: "%s()" doesn''t look like a function name. '
-                    'Setting breakpoint anyway\n' % (name,))
+                    'Setting breakpoint anyway\n' % name)
             add_breakpoint_funcname(name, i)
             return
         # "number" does the same as ":number"
@@ -633,6 +634,7 @@
         except ValueError:
             revdb.send_output('"%s": expected a line number after colon\n' % (
                 name,))
+            revdb.send_change_breakpoint(i)
             return
         filename = name[:j]
 
@@ -642,6 +644,7 @@
     if filename == '':
         frame = fetch_cur_frame()
         if frame is None:
+            revdb.send_change_breakpoint(i)
             return
         filename = frame.getcode().co_filename
     elif filename.startswith('<') and filename.endswith('>'):
@@ -649,7 +652,7 @@
     elif not filename.lower().endswith('.py'):
         # use unmodified, but warn
         revdb.send_output(
-            'Note: "%s" doesn''t look like a co_filename. '
+            'Note: "%s" doesn''t look like a Python filename. '
             'Setting breakpoint anyway\n' % (filename,))
     elif '\x00' not in filename:
         filename = rstring.assert_str0(filename)
diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
--- a/pypy/interpreter/test/test_reverse_debugging.py
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -77,13 +77,14 @@
 
     got_output = None
     got_chbkpt = None
-    if messages:
-        assert len(messages) <= 1
-        if messages[0][0] == revdb.ANSWER_TEXT:
-            got_output = messages[0][-1]
-        if messages[0][0] == revdb.ANSWER_CHBKPT:
-            assert messages[0][1] == 5
-            got_chbkpt = messages[0][-1]
+    for msg in messages:
+        if msg[0] == revdb.ANSWER_TEXT:
+            assert got_output is None
+            got_output = msg[-1]
+        elif msg[0] == revdb.ANSWER_CHBKPT:
+            assert got_chbkpt is None
+            assert msg[1] == 5
+            got_chbkpt = msg[-1]
 
     assert got_output == expected_output
     assert got_chbkpt == expected_chbkpt
@@ -92,7 +93,8 @@
     return rpath.rnormpath(rpath.rabspath(path))
 
 def test_add_breakpoint():
-    check_add_breakpoint('', expected_output="Empty breakpoint name\n")
+    check_add_breakpoint('', expected_output="Empty breakpoint name\n",
+                         expected_chbkpt='')
     check_add_breakpoint('foo42', expected_funcname="foo42")
     check_add_breakpoint('foo.bar', expected_funcname="foo.bar",
         expected_output='Note: "foo.bar()" doesn''t look like a function name.'
@@ -105,7 +107,7 @@
                          expected_fileline=('abcd', 42),
                          expected_chbkpt='abcd:42')
     check_add_breakpoint('abcd:42', expected_fileline=('abcd', 42),
-        expected_output='Note: "abcd" doesnt look like a co_filename.'
+        expected_output='Note: "abcd" doesnt look like a Python filename.'
                         ' Setting breakpoint anyway\n')
     full = fullpath('abcd.py')
     check_add_breakpoint('abcd.py:42',
@@ -113,3 +115,6 @@
                          expected_chbkpt='%s:42' % full)
     check_add_breakpoint('%s:42' % full,
                          expected_fileline=(full, 42))
+    check_add_breakpoint('42:abc',
+        expected_output='"42:abc": expected a line number after colon\n',
+        expected_chbkpt='')
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -55,7 +55,7 @@
 def send_linecache(filename, linenum, strip=True):
     send_answer(ANSWER_LINECACHE, linenum, int(strip), extra=filename)
 
-def send_change_breakpoint(breakpointnum, newtext):
+def send_change_breakpoint(breakpointnum, newtext=''):
     send_answer(ANSWER_CHBKPT, breakpointnum, extra=newtext)
 
 def current_time():
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -184,20 +184,13 @@
         new = 1
         while new in b.num2break:
             new += 1
-        if len(break_at) > 0xFFFFFF:
-            raise OverflowError("break/watchpoint too complex")
-        b.num2break[new] = (break_code +
-                            chr(len(break_at) & 0xFF) +
-                            chr((len(break_at) >> 8) & 0xFF) +
-                            chr(len(break_at) >> 16) +
-                            break_at)
+        b.set_num2break(new, break_code, break_at)
         b.sources[new] = source_expr
         if break_code == 'W':
             b.watchvalues[new] = ''
             if nids:
                 b.watchuids[new] = self.pgroup.nids_to_uids(nids)
-        kind, name = self._bp_kind(new)
-        print "%s %d added" % (kind.capitalize(), new)
+        return new
 
     def cmd_info_breakpoints(self):
         """List current breakpoints and watchpoints"""
@@ -422,7 +415,18 @@
         if not argument:
             print "Break where?"
             return
-        self._bp_new(argument, 'B', argument)
+        num = self._bp_new(argument, 'B', argument)
+        b = self.pgroup.edit_breakpoints()
+        old = b.num2break[num]
+        self.pgroup.update_breakpoints()
+        new = b.num2break.get(num)
+        if old == new:
+            print "Breakpoint %d added" % (num,)
+        elif new is None:
+            print "Breakpoint not added"
+        else:
+            kind, name = self._bp_kind(num)
+            print "Breakpoint %d added as: %s" % (num, name)
     command_b = command_break
 
     def command_delete(self, argument):
@@ -458,8 +462,9 @@
             print 'Watchpoint not added'
             return
         #
-        self._bp_new(argument, 'W', compiled_code, nids=nids)
+        new = self._bp_new(argument, 'W', compiled_code, nids=nids)
         self.pgroup.update_watch_values()
+        print "Watchpoint %d added" % (new,)
 
     def getlinecacheoutput(self, pygments_background):
         if not pygments_background or pygments_background == 'off':
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -78,6 +78,10 @@
 #    Message(ANSWER_WATCH, ok_flag, extra=result_of_expr)
 ANSWER_WATCH      = 23
 
+# sent sometimes after CMD_BREAKPOINTS:
+#    Message(ANSWER_CHBKPT, bkpt_num, extra=new_breakpoint_text)
+ANSWER_CHBKPT     = 24
+
 
 # ____________________________________________________________
 
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -60,6 +60,15 @@
         a.thread_num = self.thread_num
         return a
 
+    def set_num2break(self, new, break_code, break_at):
+        if len(break_at) > 0xFFFFFF:
+            raise OverflowError("break/watchpoint too complex")
+        self.num2break[new] = (break_code +
+                               chr(len(break_at) & 0xFF) +
+                               chr((len(break_at) >> 8) & 0xFF) +
+                               chr(len(break_at) >> 16) +
+                               break_at)
+
 
 class RecreateSubprocess(Exception):
     pass
@@ -218,6 +227,16 @@
                 sys.stdout.flush()
             elif msg.cmd == ANSWER_ATTEMPT_IO:
                 raise RecreateSubprocess
+            elif msg.cmd == ANSWER_CHBKPT and pgroup is not None:
+                # change the breakpoint definition.  Needed for
+                # ":linenum" breakpoints which must be expanded to the
+                # current file only once
+                b = pgroup.edit_breakpoints()
+                assert b.num2break[msg.arg1][0] == 'B'
+                if msg.extra:
+                    b.set_num2break(msg.arg1, 'B', msg.extra)
+                else:
+                    del b.set_num2break[msg.arg1]
             else:
                 print >> sys.stderr, "unexpected %r" % (msg,)
 
@@ -399,7 +418,7 @@
             arg2 = self.all_breakpoints.thread_num
             extra = ''.join(flat)
             self.active.send(Message(CMD_BREAKPOINTS, arg1, arg2, extra=extra))
-            self.active.expect_ready()
+            self.active.print_text_answer(pgroup=self)
         else:
             assert cmp == 1
 

From pypy.commits at gmail.com  Wed Aug 31 14:21:42 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 31 Aug 2016 11:21:42 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: add missing method override
Message-ID: <57c72036.6211c20a.f379a.6e89@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86789:2606925a333b
Date: 2016-08-31 21:18 +0300
http://bitbucket.org/pypy/pypy/changeset/2606925a333b/

Log:	add missing method override

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -341,6 +341,9 @@
     def getitemsize(self):
         return self.itemsize
 
+    def getndim(self):
+        return self.ndim
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:

From pypy.commits at gmail.com  Wed Aug 31 14:56:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 11:56:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c72847.94071c0a.1c406.b107@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86790:134d5b23ab97
Date: 2016-08-31 19:29 +0100
http://bitbucket.org/pypy/pypy/changeset/134d5b23ab97/

Log:	hg merge py3k

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -157,7 +157,6 @@
                 # poor man's x.decode('ascii', 'replace'), since it's not
                 # supported by RPython
                 if not we_are_translated():
-                    import pdb;pdb.set_trace()
                     print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
                 lst = []
                 for ch in x:

From pypy.commits at gmail.com  Wed Aug 31 14:56:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 11:56:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Quick non-fix to let translation proceed
Message-ID: <57c7284a.c62f1c0a.57e03.afb2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86791:5ddc53d46363
Date: 2016-08-31 20:04 +0100
http://bitbucket.org/pypy/pypy/changeset/5ddc53d46363/

Log:	Quick non-fix to let translation proceed

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -441,6 +441,13 @@
             self.fmt_a(w_value)
 
         def fmt_a(self, w_value):
+            if not do_unicode:
+                # - on the bytes StringFormatter, %r or %a must not call
+                #   std_wp(unicode)
+                raise NotImplementedError("FIX ME")
+                # - another problem with fmt_r = fmt_a is that on
+                #   UnicodeFormatter, repr() != ascii() sometimes
+
             from pypy.objspace.std.unicodeobject import ascii_from_object
             w_value = ascii_from_object(self.space, w_value)
             self.std_wp(self.space.unicode_w(w_value))

From pypy.commits at gmail.com  Wed Aug 31 15:10:44 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 Aug 2016 12:10:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix memoryview(<ctypes.Structure>)
Message-ID: <57c72bb4.02c41c0a.7d8c2.bd56@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86792:76de024304d4
Date: 2016-08-31 20:10 +0100
http://bitbucket.org/pypy/pypy/changeset/76de024304d4/

Log:	fix memoryview(<ctypes.Structure>)

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
             return self.value
 
     def __buffer__(self, flags):
-        return buffer(self._buffer)
+        return memoryview(self._buffer)
 
     def _get_b_base(self):
         try:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -237,7 +237,8 @@
     def _buffer(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                                   space.newint(flags))
             if space.isinstance_w(w_result, space.w_memoryview):
                 return w_result
         raise BufferInterfaceNotFound

From pypy.commits at gmail.com  Wed Aug 31 16:57:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 13:57:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix: the CPython 3.5 sre_compile.py
 produces different codes in very
Message-ID: <57c744a5.448e1c0a.28520.dca5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86793:71e54b79286e
Date: 2016-08-31 22:56 +0200
http://bitbucket.org/pypy/pypy/changeset/71e54b79286e/

Log:	Fix: the CPython 3.5 sre_compile.py produces different codes in very
	rare cases

diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -365,7 +365,9 @@
         for op1, checkerfn in unroll_char_checker:
             if op1 == op:
                 return checkerfn(ctx, ptr, ppos)
-        raise Error("next_char_ok[%d]" % op)
+        # obscure case: it should be a single char pattern, but isn't
+        # one of the opcodes in unroll_char_checker (see test_ext_opcode)
+        return sre_match(ctx, ppos, ptr, self.start_marks) is not None
 
 class AbstractUntilMatchResult(MatchResult):
 
@@ -743,7 +745,8 @@
             minptr = start + ctx.pat(ppos+1)
             if minptr > ctx.end:
                 return    # cannot match
-            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2))
+            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2),
+                                      marks)
             # when we arrive here, ptr points to the tail of the target
             # string.  check if the rest of the pattern matches,
             # and backtrack if not.
@@ -765,7 +768,7 @@
                 if minptr > ctx.end:
                     return   # cannot match
                 # count using pattern min as the maximum
-                ptr = find_repetition_end(ctx, ppos+3, ptr, min)
+                ptr = find_repetition_end(ctx, ppos+3, ptr, min, marks)
                 if ptr < minptr:
                     return   # did not match minimum number of times
 
@@ -812,7 +815,7 @@
     return True
 
 @specializectx
-def find_repetition_end(ctx, ppos, ptr, maxcount):
+def find_repetition_end(ctx, ppos, ptr, maxcount, marks):
     end = ctx.end
     ptrp1 = ptr + 1
     # First get rid of the cases where we don't have room for any match.
@@ -827,8 +830,11 @@
         if op1 == op:
             if checkerfn(ctx, ptr, ppos):
                 break
+            return ptr
     else:
-        return ptr
+        # obscure case: it should be a single char pattern, but isn't
+        # one of the opcodes in unroll_char_checker (see test_ext_opcode)
+        return general_find_repetition_end(ctx, ppos, ptr, maxcount, marks)
     # It matches at least once.  If maxcount == 1 (relatively common),
     # then we are done.
     if maxcount == 1:
@@ -846,6 +852,19 @@
     raise Error("rsre.find_repetition_end[%d]" % op)
 
 @specializectx
+def general_find_repetition_end(ctx, ppos, ptr, maxcount, marks):
+    # moved into its own JIT-opaque function
+    end = ctx.end
+    if maxcount != rsre_char.MAXREPEAT:
+        # adjust end
+        end1 = ptr + maxcount
+        if end1 <= end:
+            end = end1
+    while ptr < end and sre_match(ctx, ppos, ptr, marks) is not None:
+        ptr += 1
+    return ptr
+
+ at specializectx
 def match_ANY(ctx, ptr, ppos):   # dot wildcard.
     return not rsre_char.is_linebreak(ctx.str(ptr))
 def match_ANY_ALL(ctx, ptr, ppos):
diff --git a/rpython/rlib/rsre/test/test_ext_opcode.py b/rpython/rlib/rsre/test/test_ext_opcode.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rsre/test/test_ext_opcode.py
@@ -0,0 +1,26 @@
+"""
+Test for cases that cannot be produced using the Python 2.7 sre_compile
+module, but can be produced by other means (e.g. Python 3.5)
+"""
+
+from rpython.rlib.rsre import rsre_core
+from rpython.rlib.rsre.rsre_char import MAXREPEAT
+
+# import OPCODE_XX as XX
+for name, value in rsre_core.__dict__.items():
+    if name.startswith('OPCODE_') and isinstance(value, int):
+        globals()[name[7:]] = value
+
+
+def test_repeat_one_with_backref():
+    # Python 3.5 compiles "(.)\1*" using REPEAT_ONE instead of REPEAT:
+    # it's a valid optimization because \1 is always one character long
+    r = [MARK, 0, ANY, MARK, 1, REPEAT_ONE, 6, 0, MAXREPEAT, 
+         GROUPREF, 0, SUCCESS, SUCCESS]
+    assert rsre_core.match(r, "aaa").match_end == 3
+
+def test_min_repeat_one_with_backref():
+    # Python 3.5 compiles "(.)\1*?b" using MIN_REPEAT_ONE
+    r = [MARK, 0, ANY, MARK, 1, MIN_REPEAT_ONE, 6, 0, MAXREPEAT,
+         GROUPREF, 0, SUCCESS, LITERAL, 98, SUCCESS]
+    assert rsre_core.match(r, "aaab").match_end == 4

From pypy.commits at gmail.com  Wed Aug 31 16:58:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 13:58:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57c744ed.919a1c0a.bca81.e33b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86794:adcb5fc61bbe
Date: 2016-08-31 22:57 +0200
http://bitbucket.org/pypy/pypy/changeset/adcb5fc61bbe/

Log:	hg merge default

diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -365,7 +365,9 @@
         for op1, checkerfn in unroll_char_checker:
             if op1 == op:
                 return checkerfn(ctx, ptr, ppos)
-        raise Error("next_char_ok[%d]" % op)
+        # obscure case: it should be a single char pattern, but isn't
+        # one of the opcodes in unroll_char_checker (see test_ext_opcode)
+        return sre_match(ctx, ppos, ptr, self.start_marks) is not None
 
 class AbstractUntilMatchResult(MatchResult):
 
@@ -743,7 +745,8 @@
             minptr = start + ctx.pat(ppos+1)
             if minptr > ctx.end:
                 return    # cannot match
-            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2))
+            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2),
+                                      marks)
             # when we arrive here, ptr points to the tail of the target
             # string.  check if the rest of the pattern matches,
             # and backtrack if not.
@@ -765,7 +768,7 @@
                 if minptr > ctx.end:
                     return   # cannot match
                 # count using pattern min as the maximum
-                ptr = find_repetition_end(ctx, ppos+3, ptr, min)
+                ptr = find_repetition_end(ctx, ppos+3, ptr, min, marks)
                 if ptr < minptr:
                     return   # did not match minimum number of times
 
@@ -812,7 +815,7 @@
     return True
 
 @specializectx
-def find_repetition_end(ctx, ppos, ptr, maxcount):
+def find_repetition_end(ctx, ppos, ptr, maxcount, marks):
     end = ctx.end
     ptrp1 = ptr + 1
     # First get rid of the cases where we don't have room for any match.
@@ -827,8 +830,11 @@
         if op1 == op:
             if checkerfn(ctx, ptr, ppos):
                 break
+            return ptr
     else:
-        return ptr
+        # obscure case: it should be a single char pattern, but isn't
+        # one of the opcodes in unroll_char_checker (see test_ext_opcode)
+        return general_find_repetition_end(ctx, ppos, ptr, maxcount, marks)
     # It matches at least once.  If maxcount == 1 (relatively common),
     # then we are done.
     if maxcount == 1:
@@ -846,6 +852,19 @@
     raise Error("rsre.find_repetition_end[%d]" % op)
 
 @specializectx
+def general_find_repetition_end(ctx, ppos, ptr, maxcount, marks):
+    # moved into its own JIT-opaque function
+    end = ctx.end
+    if maxcount != rsre_char.MAXREPEAT:
+        # adjust end
+        end1 = ptr + maxcount
+        if end1 <= end:
+            end = end1
+    while ptr < end and sre_match(ctx, ppos, ptr, marks) is not None:
+        ptr += 1
+    return ptr
+
+ at specializectx
 def match_ANY(ctx, ptr, ppos):   # dot wildcard.
     return not rsre_char.is_linebreak(ctx.str(ptr))
 def match_ANY_ALL(ctx, ptr, ppos):
diff --git a/rpython/rlib/rsre/test/test_ext_opcode.py b/rpython/rlib/rsre/test/test_ext_opcode.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rsre/test/test_ext_opcode.py
@@ -0,0 +1,26 @@
+"""
+Test for cases that cannot be produced using the Python 2.7 sre_compile
+module, but can be produced by other means (e.g. Python 3.5)
+"""
+
+from rpython.rlib.rsre import rsre_core
+from rpython.rlib.rsre.rsre_char import MAXREPEAT
+
+# import OPCODE_XX as XX
+for name, value in rsre_core.__dict__.items():
+    if name.startswith('OPCODE_') and isinstance(value, int):
+        globals()[name[7:]] = value
+
+
+def test_repeat_one_with_backref():
+    # Python 3.5 compiles "(.)\1*" using REPEAT_ONE instead of REPEAT:
+    # it's a valid optimization because \1 is always one character long
+    r = [MARK, 0, ANY, MARK, 1, REPEAT_ONE, 6, 0, MAXREPEAT, 
+         GROUPREF, 0, SUCCESS, SUCCESS]
+    assert rsre_core.match(r, "aaa").match_end == 3
+
+def test_min_repeat_one_with_backref():
+    # Python 3.5 compiles "(.)\1*?b" using MIN_REPEAT_ONE
+    r = [MARK, 0, ANY, MARK, 1, MIN_REPEAT_ONE, 6, 0, MAXREPEAT,
+         GROUPREF, 0, SUCCESS, LITERAL, 98, SUCCESS]
+    assert rsre_core.match(r, "aaab").match_end == 4

From pypy.commits at gmail.com  Wed Aug 31 16:58:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 13:58:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57c744f0.0205c20a.26e9b.aab4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r86795:ca71a23cdf5e
Date: 2016-08-31 22:57 +0200
http://bitbucket.org/pypy/pypy/changeset/ca71a23cdf5e/

Log:	hg merge py3k

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
             return self.value
 
     def __buffer__(self, flags):
-        return buffer(self._buffer)
+        return memoryview(self._buffer)
 
     def _get_b_base(self):
         try:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -237,7 +237,8 @@
     def _buffer(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                                   space.newint(flags))
             if space.isinstance_w(w_result, space.w_memoryview):
                 return w_result
         raise BufferInterfaceNotFound
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -365,7 +365,9 @@
         for op1, checkerfn in unroll_char_checker:
             if op1 == op:
                 return checkerfn(ctx, ptr, ppos)
-        raise Error("next_char_ok[%d]" % op)
+        # obscure case: it should be a single char pattern, but isn't
+        # one of the opcodes in unroll_char_checker (see test_ext_opcode)
+        return sre_match(ctx, ppos, ptr, self.start_marks) is not None
 
 class AbstractUntilMatchResult(MatchResult):
 
@@ -743,7 +745,8 @@
             minptr = start + ctx.pat(ppos+1)
             if minptr > ctx.end:
                 return    # cannot match
-            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2))
+            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2),
+                                      marks)
             # when we arrive here, ptr points to the tail of the target
             # string.  check if the rest of the pattern matches,
             # and backtrack if not.
@@ -765,7 +768,7 @@
                 if minptr > ctx.end:
                     return   # cannot match
                 # count using pattern min as the maximum
-                ptr = find_repetition_end(ctx, ppos+3, ptr, min)
+                ptr = find_repetition_end(ctx, ppos+3, ptr, min, marks)
                 if ptr < minptr:
                     return   # did not match minimum number of times
 
@@ -812,7 +815,7 @@
     return True
 
 @specializectx
-def find_repetition_end(ctx, ppos, ptr, maxcount):
+def find_repetition_end(ctx, ppos, ptr, maxcount, marks):
     end = ctx.end
     ptrp1 = ptr + 1
     # First get rid of the cases where we don't have room for any match.
@@ -827,8 +830,11 @@
         if op1 == op:
             if checkerfn(ctx, ptr, ppos):
                 break
+            return ptr
     else:
-        return ptr
+        # obscure case: it should be a single char pattern, but isn't
+        # one of the opcodes in unroll_char_checker (see test_ext_opcode)
+        return general_find_repetition_end(ctx, ppos, ptr, maxcount, marks)
     # It matches at least once.  If maxcount == 1 (relatively common),
     # then we are done.
     if maxcount == 1:
@@ -846,6 +852,19 @@
     raise Error("rsre.find_repetition_end[%d]" % op)
 
 @specializectx
+def general_find_repetition_end(ctx, ppos, ptr, maxcount, marks):
+    # moved into its own JIT-opaque function
+    end = ctx.end
+    if maxcount != rsre_char.MAXREPEAT:
+        # adjust end
+        end1 = ptr + maxcount
+        if end1 <= end:
+            end = end1
+    while ptr < end and sre_match(ctx, ppos, ptr, marks) is not None:
+        ptr += 1
+    return ptr
+
+ at specializectx
 def match_ANY(ctx, ptr, ppos):   # dot wildcard.
     return not rsre_char.is_linebreak(ctx.str(ptr))
 def match_ANY_ALL(ctx, ptr, ppos):
diff --git a/rpython/rlib/rsre/test/test_ext_opcode.py b/rpython/rlib/rsre/test/test_ext_opcode.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rsre/test/test_ext_opcode.py
@@ -0,0 +1,26 @@
+"""
+Test for cases that cannot be produced using the Python 2.7 sre_compile
+module, but can be produced by other means (e.g. Python 3.5)
+"""
+
+from rpython.rlib.rsre import rsre_core
+from rpython.rlib.rsre.rsre_char import MAXREPEAT
+
+# import OPCODE_XX as XX
+for name, value in rsre_core.__dict__.items():
+    if name.startswith('OPCODE_') and isinstance(value, int):
+        globals()[name[7:]] = value
+
+
+def test_repeat_one_with_backref():
+    # Python 3.5 compiles "(.)\1*" using REPEAT_ONE instead of REPEAT:
+    # it's a valid optimization because \1 is always one character long
+    r = [MARK, 0, ANY, MARK, 1, REPEAT_ONE, 6, 0, MAXREPEAT, 
+         GROUPREF, 0, SUCCESS, SUCCESS]
+    assert rsre_core.match(r, "aaa").match_end == 3
+
+def test_min_repeat_one_with_backref():
+    # Python 3.5 compiles "(.)\1*?b" using MIN_REPEAT_ONE
+    r = [MARK, 0, ANY, MARK, 1, MIN_REPEAT_ONE, 6, 0, MAXREPEAT,
+         GROUPREF, 0, SUCCESS, LITERAL, 98, SUCCESS]
+    assert rsre_core.match(r, "aaab").match_end == 4

From pypy.commits at gmail.com  Wed Aug 31 17:00:19 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 31 Aug 2016 14:00:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: improve method calls on oldstyle classes
Message-ID: <57c74563.031dc20a.a3c9a.a7d2@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86796:573ad1ed00c6
Date: 2016-08-31 21:59 +0100
http://bitbucket.org/pypy/pypy/changeset/573ad1ed00c6/

Log:	improve method calls on oldstyle classes

diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -38,6 +38,8 @@
 
 
 class W_ClassObject(W_Root):
+    _immutable_fields_ = ['bases_w?[*]', 'w_dict?']
+
     def __init__(self, space, w_name, bases, w_dict):
         self.name = space.str_w(w_name)
         make_sure_not_resized(bases)
@@ -75,6 +77,7 @@
                             "__bases__ items must be classes")
         self.bases_w = bases_w
 
+    @jit.unroll_safe
     def is_subclass_of(self, other):
         assert isinstance(other, W_ClassObject)
         if self is other:
@@ -313,7 +316,7 @@
         # This method ignores the instance dict and the __getattr__.
         # Returns None if not found.
         assert isinstance(name, str)
-        w_value = self.w_class.lookup(space, name)
+        w_value = jit.promote(self.w_class).lookup(space, name)
         if w_value is None:
             return None
         w_descr_get = space.lookup(w_value, '__get__')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -187,6 +187,43 @@
 
         """)
 
+    def test_oldstyle_methcall(self):
+        def main():
+            def g(): pass
+            class A:
+                def f(self):
+                    return self.x + 1
+            class I(A):
+                pass
+            class J(I):
+                pass
+
+
+            class B(J):
+                def __init__(self, x):
+                    self.x = x
+
+            i = 0
+            b = B(1)
+            while i < 1000:
+                g()
+                v = b.f() # ID: meth
+                i += v
+            return i
+
+        log = self.run(main, [], threshold=80)
+        loop, = log.loops_by_filename(self.filepath, is_entry_bridge=True)
+        assert loop.match_by_id('meth',
+        '''
+    guard_nonnull_class(p18, ..., descr=...)
+    p52 = getfield_gc_r(p18, descr=...) # read map
+    guard_value(p52, ConstPtr(ptr53), descr=...)
+    p54 = getfield_gc_r(p18, descr=...) # read class
+    guard_value(p54, ConstPtr(ptr55), descr=...)
+    p56 = force_token() # done
+        ''')
+
+
     def test_oldstyle_newstyle_mix(self):
         def main():
             class A:

From pypy.commits at gmail.com  Wed Aug 31 17:00:21 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 31 Aug 2016 14:00:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <57c74565.c19d1c0a.11b09.dee5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86797:ec5bb4dd8a06
Date: 2016-08-31 22:07 +0100
http://bitbucket.org/pypy/pypy/changeset/ec5bb4dd8a06/

Log:	merge heads

diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -365,7 +365,9 @@
         for op1, checkerfn in unroll_char_checker:
             if op1 == op:
                 return checkerfn(ctx, ptr, ppos)
-        raise Error("next_char_ok[%d]" % op)
+        # obscure case: it should be a single char pattern, but isn't
+        # one of the opcodes in unroll_char_checker (see test_ext_opcode)
+        return sre_match(ctx, ppos, ptr, self.start_marks) is not None
 
 class AbstractUntilMatchResult(MatchResult):
 
@@ -743,7 +745,8 @@
             minptr = start + ctx.pat(ppos+1)
             if minptr > ctx.end:
                 return    # cannot match
-            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2))
+            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2),
+                                      marks)
             # when we arrive here, ptr points to the tail of the target
             # string.  check if the rest of the pattern matches,
             # and backtrack if not.
@@ -765,7 +768,7 @@
                 if minptr > ctx.end:
                     return   # cannot match
                 # count using pattern min as the maximum
-                ptr = find_repetition_end(ctx, ppos+3, ptr, min)
+                ptr = find_repetition_end(ctx, ppos+3, ptr, min, marks)
                 if ptr < minptr:
                     return   # did not match minimum number of times
 
@@ -812,7 +815,7 @@
     return True
 
 @specializectx
-def find_repetition_end(ctx, ppos, ptr, maxcount):
+def find_repetition_end(ctx, ppos, ptr, maxcount, marks):
     end = ctx.end
     ptrp1 = ptr + 1
     # First get rid of the cases where we don't have room for any match.
@@ -827,8 +830,11 @@
         if op1 == op:
             if checkerfn(ctx, ptr, ppos):
                 break
+            return ptr
     else:
-        return ptr
+        # obscure case: it should be a single char pattern, but isn't
+        # one of the opcodes in unroll_char_checker (see test_ext_opcode)
+        return general_find_repetition_end(ctx, ppos, ptr, maxcount, marks)
     # It matches at least once.  If maxcount == 1 (relatively common),
     # then we are done.
     if maxcount == 1:
@@ -846,6 +852,19 @@
     raise Error("rsre.find_repetition_end[%d]" % op)
 
 @specializectx
+def general_find_repetition_end(ctx, ppos, ptr, maxcount, marks):
+    # moved into its own JIT-opaque function
+    end = ctx.end
+    if maxcount != rsre_char.MAXREPEAT:
+        # adjust end
+        end1 = ptr + maxcount
+        if end1 <= end:
+            end = end1
+    while ptr < end and sre_match(ctx, ppos, ptr, marks) is not None:
+        ptr += 1
+    return ptr
+
+ at specializectx
 def match_ANY(ctx, ptr, ppos):   # dot wildcard.
     return not rsre_char.is_linebreak(ctx.str(ptr))
 def match_ANY_ALL(ctx, ptr, ppos):
diff --git a/rpython/rlib/rsre/test/test_ext_opcode.py b/rpython/rlib/rsre/test/test_ext_opcode.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rsre/test/test_ext_opcode.py
@@ -0,0 +1,26 @@
+"""
+Test for cases that cannot be produced using the Python 2.7 sre_compile
+module, but can be produced by other means (e.g. Python 3.5)
+"""
+
+from rpython.rlib.rsre import rsre_core
+from rpython.rlib.rsre.rsre_char import MAXREPEAT
+
+# import OPCODE_XX as XX
+for name, value in rsre_core.__dict__.items():
+    if name.startswith('OPCODE_') and isinstance(value, int):
+        globals()[name[7:]] = value
+
+
+def test_repeat_one_with_backref():
+    # Python 3.5 compiles "(.)\1*" using REPEAT_ONE instead of REPEAT:
+    # it's a valid optimization because \1 is always one character long
+    r = [MARK, 0, ANY, MARK, 1, REPEAT_ONE, 6, 0, MAXREPEAT, 
+         GROUPREF, 0, SUCCESS, SUCCESS]
+    assert rsre_core.match(r, "aaa").match_end == 3
+
+def test_min_repeat_one_with_backref():
+    # Python 3.5 compiles "(.)\1*?b" using MIN_REPEAT_ONE
+    r = [MARK, 0, ANY, MARK, 1, MIN_REPEAT_ONE, 6, 0, MAXREPEAT,
+         GROUPREF, 0, SUCCESS, LITERAL, 98, SUCCESS]
+    assert rsre_core.match(r, "aaab").match_end == 4

From pypy.commits at gmail.com  Wed Aug 31 17:43:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 14:43:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: haaaack on top of previous hacks
Message-ID: <57c74f97.031dc20a.a3c9a.b436@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86798:3f56c9ee1d6e
Date: 2016-08-31 23:43 +0200
http://bitbucket.org/pypy/pypy/changeset/3f56c9ee1d6e/

Log:	haaaack on top of previous hacks

diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -98,8 +98,15 @@
         try:
             ctypes.CDLL(name)
         except OSError as e:
+            # common case: ctypes fails too, with the real dlerror()
+            # message in str(e).  Return that error message.
             return str(e)
         else:
+            # uncommon case: may happen if 'name' is a linker script
+            # (which the C-level dlopen() can't handle) and we are
+            # directly running on pypy (whose implementation of ctypes
+            # or cffi will resolve linker scripts).  In that case, 
+            # unsure what we can do.
             return ("opening %r with ctypes.CDLL() works, "
                     "but not with c_dlopen()??" % (name,))
 
@@ -160,6 +167,13 @@
             mode = _dlopen_default_mode()
         elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
+        #
+        # haaaack for 'pypy py.test -A' if libm.so is a linker script
+        # (see reason in _dlerror_on_dlopen_untranslated())
+        if not we_are_translated() and platform.name == "linux":
+            if rffi.charp2str(name) == 'libm.so':
+                name = rffi.str2charp('libm.so.6', track_allocation=False)
+        #
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
         if not res:
             if not we_are_translated():

From pypy.commits at gmail.com  Wed Aug 31 17:51:42 2016
From: pypy.commits at gmail.com (stefanor)
Date: Wed, 31 Aug 2016 14:51:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: name can be None
Message-ID: <57c7516e.09afc20a.e370d.b0ad@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r86799:e075b4dfeb37
Date: 2016-08-31 14:51 -0700
http://bitbucket.org/pypy/pypy/changeset/e075b4dfeb37/

Log:	name can be None

diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -171,7 +171,7 @@
         # haaaack for 'pypy py.test -A' if libm.so is a linker script
         # (see reason in _dlerror_on_dlopen_untranslated())
         if not we_are_translated() and platform.name == "linux":
-            if rffi.charp2str(name) == 'libm.so':
+            if name and rffi.charp2str(name) == 'libm.so':
                 name = rffi.str2charp('libm.so.6', track_allocation=False)
         #
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))

From pypy.commits at gmail.com  Wed Aug 31 19:04:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 16:04:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add a failing (skipped) whitebox test
 and a test about ctypes that fails
Message-ID: <57c76289.8f8e1c0a.c5d55.f980@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86800:191a4cce5363
Date: 2016-09-01 01:04 +0200
http://bitbucket.org/pypy/pypy/changeset/191a4cce5363/

Log:	Add a failing (skipped) whitebox test and a test about ctypes that
	fails on -A (xfailed)

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
@@ -195,6 +195,29 @@
             _fields_ = [('t', enum)]
         assert isinstance(S().t, enum)
 
+    def test_no_missing_shape_to_ffi_type(self):
+        # whitebox test
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("only for pypy's ctypes")
+        skip("re-enable after adding 'g' to _shape_to_ffi_type.typemap, "
+             "which I think needs fighting all the way up from "
+             "rpython.rlib.libffi")
+        from _ctypes.basics import _shape_to_ffi_type
+        from _rawffi import Array
+        for i in range(1, 256):
+            try:
+                Array(chr(i))
+            except ValueError:
+                pass
+            else:
+                assert chr(i) in _shape_to_ffi_type.typemap
+
+    @py.test.mark.xfail
+    def test_pointer_to_long_double(self):
+        import ctypes
+        ctypes.POINTER(ctypes.c_longdouble)
+
 ##    def test_perf(self):
 ##        check_perf()
 
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -47,6 +47,8 @@
         cls.ulonglong = clibffi.cast_type_to_ffitype(rffi.ULONGLONG)
         cls.signed = clibffi.cast_type_to_ffitype(rffi.SIGNED)
         cls.wchar_t = clibffi.cast_type_to_ffitype(lltype.UniChar)
+        # XXX long double support: clibffi.ffi_type_longdouble, but then
+        # XXX fix the whole rest of this file to add a case for long double
         del cls._import
 
     @staticmethod

From pypy.commits at gmail.com  Wed Aug 31 19:20:45 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 31 Aug 2016 16:20:45 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Assertion and add a proper
 test
Message-ID: <57c7664d.88cb1c0a.55ab1.fda2@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86801:b46132df6a05
Date: 2016-08-31 19:19 -0400
http://bitbucket.org/pypy/pypy/changeset/b46132df6a05/

Log:	Assertion and add a proper test

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -552,10 +552,10 @@
         """
         Generate the guards and add state information for unifying a virtual
         object with a non-virtual. This involves forcing the object in the
-        event that unifcation can succeed. Since virtual objects cannot be null,
+        event that unification can succeed. Since virtual objects cannot be null,
         this method need only check that the virtual object has the expected type.
         """
-        assert isinstance(other, VirtualStateInfo)
+        assert state.force_boxes and isinstance(other, VirtualStateInfo)
 
         if self.level == LEVEL_CONSTANT:
             raise VirtualStatesCantMatch(
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4508,32 +4508,3 @@
             return i
         self.meta_interp(f, [])
 
-    def test_loop_unroll_bug(self):
-        driver = JitDriver(greens=[], reds=['acc', 'i', 'val'])
-        class X(object):
-            # _immutable_ = True
-            def __init__(self, v):
-                self.v = v
-
-        class Box(object):
-            def __init__(self, v):
-                self.unbox = v
-
-        const = Box(X(5))
-        def f(v):
-            val   = X(0)
-            acc   = 0
-            i     = 0
-            const.unbox = X(5)
-            while i < 100:
-                driver.can_enter_jit(acc=acc, i=i, val=val)
-                driver.jit_merge_point(acc=acc, i=i, val=val)
-                acc += val.v
-                if i & 0b100 == 0:
-                    val = const.unbox
-                else:
-                    val = X(i)
-                i += 1
-            return acc
-        result = self.meta_interp(f, [10])
-        # import pdb; pdb.set_trace()
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1,8 +1,8 @@
 import py
-from rpython.rlib.jit import JitDriver, promote, dont_look_inside
+from rpython.rlib.jit import JitDriver, promote, dont_look_inside, set_param
 from rpython.rlib.objectmodel import compute_unique_id
 from rpython.jit.codewriter.policy import StopAtXPolicy
-from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.test.support import LLJitMixin, get_stats
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper import rclass
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -965,6 +965,47 @@
         self.check_aborted_count(0)
         self.check_target_token_count(4)
 
+    def test_avoid_preamble(self):
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        class Box(object):
+            def __init__(self, v):
+                self.unbox = v
+
+        mask = -2
+        const = Box(X(5))
+        def f():
+            # Prevent all retracing of side exits. Ensures that the unroll
+            # optimizer will attempt to jump to either the preamble or loop.
+            set_param(driver, 'retrace_limit', -1)
+            set_param(driver, 'threshold', 1)
+            val   = X(0)
+            i     = 0
+            const.unbox = X(5)
+            while i < 17:
+                driver.can_enter_jit(i=i, val=val)
+                driver.jit_merge_point(i=i, val=val)
+                # Logical & rather than comparison to confuse range analysis.
+                # Test only succeeds on the first 2 iterations
+                if i & -2 == 0:
+                    val = const.unbox
+                else:
+                    val = X(i)
+                i += 1
+            return 0
+
+        self.meta_interp(f, [])
+
+        # With retracing disable, there will be one optimized loop expecting a
+        # non-virtual X object. The side exit creates a virtual object which must
+        # be allocated to jump to the optimized trace.
+        self.check_resops(jump=3, label=2, new_with_vtable=2)
+        self.check_target_token_count(2)
+        self.check_trace_count(3)
+
 
 class VirtualMiscTests:
     def test_multiple_equal_virtuals(self):

From pypy.commits at gmail.com  Wed Aug 31 19:22:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 16:22:56 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fixes fixes fixes
Message-ID: <57c766d0.822e1c0a.bb2f3.497d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86802:799bea7b4905
Date: 2016-09-01 00:31 +0100
http://bitbucket.org/pypy/pypy/changeset/799bea7b4905/

Log:	fixes fixes fixes

	now after "b FILENAME:LINENO", will break whenever we reach the
	LINENO in a code object with co_filename == (optional
	prefix/)FILENAME. Should all be done efficiently enough with
	multiple levels of caching.

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -1,5 +1,5 @@
 import sys
-from rpython.rlib import revdb, rpath, rstring
+from rpython.rlib import revdb
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
@@ -15,6 +15,7 @@
     breakpoint_stack_id = 0
     breakpoint_funcnames = None
     breakpoint_filelines = None
+    breakpoint_by_file = None
     breakpoint_version = 0
     printed_objects = {}
     metavars = []
@@ -131,7 +132,7 @@
             call_stop_point_at_line = False
     #
     if call_stop_point_at_line:
-        if dbstate.breakpoint_filelines is not None:
+        if dbstate.breakpoint_by_file is not None:
             check_and_trigger_bkpt(frame.pycode, cur)
         stop_point_activate()
         cur += 1
@@ -214,6 +215,7 @@
                 lastlineno = lineno
             addr += byte_incr
         lineno += line_incr
+        p += 2
     if lineno != lastlineno:
         result.append((addr, lineno))
     return result
@@ -545,6 +547,8 @@
     # mapping {opindex: bkpt_num}.  This cache is updated when the
     # version in 'pycode.co_revdb_bkpt_version' does not match
     # 'dbstate.breakpoint_version' any more.
+    #
+    # IMPORTANT: no object allocation here, outside update_bkpt_cache!
     if pycode.co_revdb_bkpt_version != dbstate.breakpoint_version:
         update_bkpt_cache(pycode)
     cache = pycode.co_revdb_bkpt_cache
@@ -552,33 +556,46 @@
         revdb.breakpoint(cache[opindex])
 
 def update_bkpt_cache(pycode):
-    # dbstate.breakpoint_filelines == {'normfilename': {lineno: bkpt_num}}
+    # initialized by command_breakpoints():
+    #     dbstate.breakpoint_filelines == [('FILENAME', lineno, bkpt_num)]
+    # computed lazily (here, first half of the logic):
+    #     dbstate.breakpoint_by_file == {'co_filename': {lineno: bkpt_num}}
+    # the goal is to set:
+    #     pycode.co_revdb_bkpt_cache == {opindex: bkpt_num}
+    #
+    prev_state = revdb.watch_save_state(force=True)
+    # ^^^ the object allocations done in this function should not count!
+
     co_filename = pycode.co_filename
     try:
-        linenos = dbstate.breakpoint_filelines[co_filename]
+        linenos = dbstate.breakpoint_by_file[co_filename]
     except KeyError:
-        # normalize co_filename, and assigns the {lineno: bkpt_num} dict
-        # back over the original key, to avoid calling rabspath/rnormpath
-        # again the next time
-        co_filename = rstring.assert_str0(co_filename)
-        normfilename = rpath.rabspath(co_filename)
-        normfilename = rpath.rnormpath(normfilename)
-        linenos = dbstate.breakpoint_filelines.get(normfilename, None)
-        dbstate.breakpoint_filelines[co_filename] = linenos
-    #
+        linenos = None
+        match = co_filename.upper()    # ignore cAsE in filename matching
+        for filename, lineno, bkpt_num in dbstate.breakpoint_filelines:
+            if match.endswith(filename) and (
+                    len(match) == len(filename) or
+                    match[-len(filename)-1] in '/\\'):    # a valid prefix
+                if linenos is None:
+                    linenos = {}
+                linenos[lineno] = bkpt_num
+        dbstate.breakpoint_by_file[co_filename] = linenos
+
     newcache = None
     if linenos is not None:
         # parse co_lnotab to figure out the opindexes that correspond
-        # to the marked line numbers.
+        # to the marked line numbers.  here, linenos == {lineno: bkpt_num}
         for addr, lineno in find_line_starts(pycode):
             if lineno in linenos:
                 if newcache is None:
                     newcache = {}
                 newcache[addr] = linenos[lineno]
-    #
+
     pycode.co_revdb_bkpt_cache = newcache
     pycode.co_revdb_bkpt_version = dbstate.breakpoint_version
 
+    revdb.watch_restore_state(prev_state)
+
 
 def valid_identifier(s):
     if not s:
@@ -596,10 +613,14 @@
     dbstate.breakpoint_funcnames[name] = i
 
 def add_breakpoint_fileline(filename, lineno, i):
+    # dbstate.breakpoint_filelines is just a list of (FILENAME, lineno, i).
+    # dbstate.breakpoint_by_file is {co_filename: {lineno: i}}, but
+    # computed lazily when we encounter a code object with the given
+    # co_filename.  Any suffix 'filename' matches 'co_filename'.
     if dbstate.breakpoint_filelines is None:
-        dbstate.breakpoint_filelines = {}
-    linenos = dbstate.breakpoint_filelines.setdefault(filename, {})
-    linenos[lineno] = i
+        dbstate.breakpoint_filelines = []
+        dbstate.breakpoint_by_file = {}
+    dbstate.breakpoint_filelines.append((filename.upper(), lineno, i))
 
 def add_breakpoint(name, i):
     # if it is empty, complain
@@ -654,10 +675,6 @@
         revdb.send_output(
             'Note: "%s" doesn''t look like a Python filename. '
             'Setting breakpoint anyway\n' % (filename,))
-    elif '\x00' not in filename:
-        filename = rstring.assert_str0(filename)
-        filename = rpath.rabspath(filename)
-        filename = rpath.rnormpath(filename)
 
     add_breakpoint_fileline(filename, lineno, i)
     name = '%s:%d' % (filename, lineno)
@@ -670,6 +687,7 @@
     revdb.set_thread_breakpoint(cmd.c_arg2)
     dbstate.breakpoint_funcnames = None
     dbstate.breakpoint_filelines = None
+    dbstate.breakpoint_by_file = None
     dbstate.breakpoint_version += 1
     watch_progs = []
     with non_standard_code:
diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
--- a/pypy/interpreter/test/test_reverse_debugging.py
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -1,7 +1,7 @@
 import dis
 from pypy.interpreter.reverse_debugging import *
 from pypy.interpreter import reverse_debugging
-from rpython.rlib import revdb, rpath
+from rpython.rlib import revdb
 from hypothesis import given, strategies, example
 
 
@@ -73,7 +73,7 @@
         assert dbstate.breakpoint_filelines is None
     else:
         filename, lineno = expected_fileline
-        assert dbstate.breakpoint_filelines == {filename: {lineno: 5}}
+        assert dbstate.breakpoint_filelines == [(filename.upper(), lineno, 5)]
 
     got_output = None
     got_chbkpt = None
@@ -89,9 +89,6 @@
     assert got_output == expected_output
     assert got_chbkpt == expected_chbkpt
 
-def fullpath(path):
-    return rpath.rnormpath(rpath.rabspath(path))
-
 def test_add_breakpoint():
     check_add_breakpoint('', expected_output="Empty breakpoint name\n",
                          expected_chbkpt='')
@@ -109,12 +106,8 @@
     check_add_breakpoint('abcd:42', expected_fileline=('abcd', 42),
         expected_output='Note: "abcd" doesnt look like a Python filename.'
                         ' Setting breakpoint anyway\n')
-    full = fullpath('abcd.py')
     check_add_breakpoint('abcd.py:42',
-                         expected_fileline=(full, 42),
-                         expected_chbkpt='%s:42' % full)
-    check_add_breakpoint('%s:42' % full,
-                         expected_fileline=(full, 42))
+                         expected_fileline=('abcd.py', 42))
     check_add_breakpoint('42:abc',
         expected_output='"42:abc": expected a line number after colon\n',
         expected_chbkpt='')
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -132,8 +132,8 @@
     ll_callback = llhelper(_CALLBACK_GCREF_FNPTR, callback)
     llop.revdb_track_object(lltype.Void, unique_id, ll_callback)
 
-def watch_save_state():
-    return llop.revdb_watch_save_state(lltype.Bool)
+def watch_save_state(force=False):
+    return llop.revdb_watch_save_state(lltype.Bool, force)
 
 def watch_restore_state(any_watch_point):
     llop.revdb_watch_restore_state(lltype.Void, any_watch_point)
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -51,6 +51,7 @@
                 print 'KeyboardInterrupt: restoring state at time %d...' % (
                     rtime,)
                 self.pgroup.recreate_subprocess(rtime)
+                print "(type 'q' or Ctrl-D to quit)"
                 self.last_command = ''
                 self.previous_thread = '?'
                 self.previous_time = '?'
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -205,9 +205,9 @@
 #define OP_REVDB_TRACK_OBJECT(uid, callback, r)                         \
     rpy_reverse_db_track_object(uid, callback)
 
-#define OP_REVDB_WATCH_SAVE_STATE(r)   do {                             \
+#define OP_REVDB_WATCH_SAVE_STATE(force, r)   do {                      \
         r = rpy_revdb.watch_enabled;                                    \
-        if (r) rpy_reverse_db_watch_save_state();                       \
+        if ((force) || r) rpy_reverse_db_watch_save_state();            \
     } while (0)
 
 #define OP_REVDB_WATCH_RESTORE_STATE(any_watch_point, r)                \

From pypy.commits at gmail.com  Wed Aug 31 19:49:30 2016
From: pypy.commits at gmail.com (stefanor)
Date: Wed, 31 Aug 2016 16:49:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: Bump recursionlimit,
 for translating with cpython
Message-ID: <57c76d0a.c398c20a.9932f.cf3e@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r86803:e94a1150b191
Date: 2016-08-31 16:41 -0700
http://bitbucket.org/pypy/pypy/changeset/e94a1150b191/

Log:	Bump recursionlimit, for translating with cpython

diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -213,6 +213,7 @@
         log.WARNING(warning)
 
 def main():
+    sys.setrecursionlimit(2000)  # PyPy can't translate within cpython's 1k limit
     targetspec_dic, translateconfig, config, args = parse_options_and_load_target()
     from rpython.translator import translator
     from rpython.translator import driver

From pypy.commits at gmail.com  Wed Aug 31 19:49:32 2016
From: pypy.commits at gmail.com (stefanor)
Date: Wed, 31 Aug 2016 16:49:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Avoid blowing up with results that are
 ~0 but <0 due to floating point imprecision
Message-ID: <57c76d0c.e97ac20a.1eabd.cde0@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r86804:447f36f19521
Date: 2016-08-31 16:48 -0700
http://bitbucket.org/pypy/pypy/changeset/447f36f19521/

Log:	Avoid blowing up with results that are ~0 but <0 due to floating
	point imprecision

diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py
--- a/rpython/translator/backendopt/inline.py
+++ b/rpython/translator/backendopt/inline.py
@@ -532,8 +532,7 @@
         return sys.maxint
     else:
         res = Solution[blockmap[graph.startblock]]
-        assert res >= 0
-        return res
+        return max(res, 0.0)
 
 def static_instruction_count(graph):
     count = 0

From pypy.commits at gmail.com  Wed Aug 31 21:09:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 18:09:46 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: more fixes
Message-ID: <57c77fda.11051c0a.a4c42.1194@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86805:e35785260055
Date: 2016-09-01 02:18 +0100
http://bitbucket.org/pypy/pypy/changeset/e35785260055/

Log:	more fixes

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -132,9 +132,7 @@
             call_stop_point_at_line = False
     #
     if call_stop_point_at_line:
-        if dbstate.breakpoint_by_file is not None:
-            check_and_trigger_bkpt(frame.pycode, cur)
-        stop_point_activate()
+        stop_point_activate(pycode=frame.pycode, opindex=cur)
         cur += 1
         ch = ord(co_revdb_linestarts[cur])
     #
@@ -232,23 +230,33 @@
 non_standard_code = NonStandardCode()
 
 
-def stop_point_activate(place=0):
+def stop_point_activate(place=0, pycode=None, opindex=-1):
     if revdb.watch_save_state():
         any_watch_point = False
+        # ^^ this flag is set to True if we must continue to enter this
+        # block of code.  If it is still False for watch_restore_state()
+        # below, then future watch_save_state() will return False too---
+        # until the next time revdb.c:set_revdb_breakpoints() is called.
         space = dbstate.space
         with non_standard_code:
-            for prog, watch_id, expected in dbstate.watch_progs:
+            watch_id = -1
+            if dbstate.breakpoint_by_file is not None:
                 any_watch_point = True
-                try:
-                    got = _run_watch(space, prog)
-                except OperationError as e:
-                    got = e.errorstr(space)
-                except Exception:
-                    break
-                if got != expected:
-                    break
-            else:
-                watch_id = -1
+                if pycode is not None:
+                    watch_id = check_and_trigger_bkpt(pycode, opindex)
+            if watch_id == -1:
+                for prog, watch_id, expected in dbstate.watch_progs:
+                    any_watch_point = True
+                    try:
+                        got = _run_watch(space, prog)
+                    except OperationError as e:
+                        got = e.errorstr(space)
+                    except Exception:
+                        break
+                    if got != expected:
+                        break
+                else:
+                    watch_id = -1
         revdb.watch_restore_state(any_watch_point)
         if watch_id != -1:
             revdb.breakpoint(watch_id)
@@ -547,13 +555,13 @@
     # mapping {opindex: bkpt_num}.  This cache is updated when the
     # version in 'pycode.co_revdb_bkpt_version' does not match
     # 'dbstate.breakpoint_version' any more.
-    #
-    # IMPORTANT: no object allocation here, outside update_bkpt_cache!
     if pycode.co_revdb_bkpt_version != dbstate.breakpoint_version:
         update_bkpt_cache(pycode)
     cache = pycode.co_revdb_bkpt_cache
     if cache is not None and opindex in cache:
-        revdb.breakpoint(cache[opindex])
+        return cache[opindex]
+    else:
+        return -1
 
 def update_bkpt_cache(pycode):
     # initialized by command_breakpoints():
@@ -562,9 +570,6 @@
     #     dbstate.breakpoint_by_file == {'co_filename': {lineno: bkpt_num}}
     # the goal is to set:
     #     pycode.co_revdb_bkpt_cache == {opindex: bkpt_num}
-    #
-    prev_state = revdb.watch_save_state(force=True)
-    # ^^^ the object allocations done in this function should not count!
 
     co_filename = pycode.co_filename
     try:
@@ -594,8 +599,6 @@
     pycode.co_revdb_bkpt_cache = newcache
     pycode.co_revdb_bkpt_version = dbstate.breakpoint_version
 
-    revdb.watch_restore_state(prev_state)
-
 
 def valid_identifier(s):
     if not s:

From pypy.commits at gmail.com  Wed Aug 31 21:22:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 Aug 2016 18:22:39 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: UI tweaks
Message-ID: <57c782df.d8011c0a.777e7.0da6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86806:abac6184118c
Date: 2016-09-01 02:31 +0100
http://bitbucket.org/pypy/pypy/changeset/abac6184118c/

Log:	UI tweaks

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -643,11 +643,18 @@
         try:
             lineno = int(name)
         except ValueError:
+            if name.endswith('()'):
+                n = len(name) - 2
+                assert n >= 0
+                name = name[:n]
             if not valid_identifier(name):
                 revdb.send_output(
                     'Note: "%s()" doesn''t look like a function name. '
                     'Setting breakpoint anyway\n' % name)
             add_breakpoint_funcname(name, i)
+            name += '()'
+            if name != original_name:
+                revdb.send_change_breakpoint(i, name)
             return
         # "number" does the same as ":number"
         filename = ''
@@ -656,8 +663,7 @@
         try:
             lineno = int(name[j+1:])
         except ValueError:
-            revdb.send_output('"%s": expected a line number after colon\n' % (
-                name,))
+            revdb.send_output('expected a line number after colon\n')
             revdb.send_change_breakpoint(i)
             return
         filename = name[:j]
diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
--- a/pypy/interpreter/test/test_reverse_debugging.py
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -92,10 +92,13 @@
 def test_add_breakpoint():
     check_add_breakpoint('', expected_output="Empty breakpoint name\n",
                          expected_chbkpt='')
-    check_add_breakpoint('foo42', expected_funcname="foo42")
+    check_add_breakpoint('foo42', expected_funcname="foo42",
+                         expected_chbkpt="foo42()")
+    check_add_breakpoint('foo42()', expected_funcname="foo42")
     check_add_breakpoint('foo.bar', expected_funcname="foo.bar",
         expected_output='Note: "foo.bar()" doesn''t look like a function name.'
-                        ' Setting breakpoint anyway\n')
+                        ' Setting breakpoint anyway\n',
+        expected_chbkpt="foo.bar()")
     check_add_breakpoint('<foo.bar>', expected_funcname="<foo.bar>")
     check_add_breakpoint('42', curfilename='abcd',
                          expected_fileline=('abcd', 42),
@@ -109,5 +112,5 @@
     check_add_breakpoint('abcd.py:42',
                          expected_fileline=('abcd.py', 42))
     check_add_breakpoint('42:abc',
-        expected_output='"42:abc": expected a line number after colon\n',
+        expected_output='expected a line number after colon\n',
         expected_chbkpt='')
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -417,23 +417,27 @@
             print "Break where?"
             return
         num = self._bp_new(argument, 'B', argument)
+        self.pgroup.update_breakpoints()
         b = self.pgroup.edit_breakpoints()
-        old = b.num2break[num]
-        self.pgroup.update_breakpoints()
-        new = b.num2break.get(num)
-        if old == new:
-            print "Breakpoint %d added" % (num,)
-        elif new is None:
+        if num not in b.num2break:
             print "Breakpoint not added"
         else:
             kind, name = self._bp_kind(num)
-            print "Breakpoint %d added as: %s" % (num, name)
+            print "Breakpoint %d added: %s" % (num, name)
     command_b = command_break
 
     def command_delete(self, argument):
         """Delete a breakpoint/watchpoint"""
-        arg = int(argument)
         b = self.pgroup.edit_breakpoints()
+        try:
+            arg = int(argument)
+        except ValueError:
+            for arg in b.num2break:
+                if self._bp_kind(arg)[1] == argument:
+                    break
+            else:
+                print "No such breakpoint/watchpoint: %s" % (argument,)
+                return
         if arg not in b.num2break:
             print "No breakpoint/watchpoint number %d" % (arg,)
         else:
@@ -443,6 +447,7 @@
             b.watchvalues.pop(arg, '')
             b.watchuids.pop(arg, '')
             print "%s %d deleted: %s" % (kind.capitalize(), arg, name)
+    command_del = command_delete
 
     def command_watch(self, argument):
         """Add a watchpoint (use $NUM in the expression to watch)"""
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -236,7 +236,7 @@
                 if msg.extra:
                     b.set_num2break(msg.arg1, 'B', msg.extra)
                 else:
-                    del b.set_num2break[msg.arg1]
+                    del b.num2break[msg.arg1]
             else:
                 print >> sys.stderr, "unexpected %r" % (msg,)